You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2017/08/09 03:49:56 UTC
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 24 - Still
Unstable
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/24/
5 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
7 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest: 1) Thread[id=34695, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 2) Thread[id=34696, name=zkCallback-1779-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=34694, name=StoppableCommitThread-SendThread(127.0.0.1:59721), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 4) Thread[id=34784, name=zkCallback-1779-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=34672, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354) at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1126) at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1112) at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:647) at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1227) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:524) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:544) at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:49) 6) Thread[id=34785, name=zkCallback-1779-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=34783, name=zkCallback-1779-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 7 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:
1) Thread[id=34695, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
2) Thread[id=34696, name=zkCallback-1779-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=34694, name=StoppableCommitThread-SendThread(127.0.0.1:59721), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
4) Thread[id=34784, name=zkCallback-1779-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=34672, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1126)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1112)
at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:647)
at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1227)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:524)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:544)
at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:49)
6) Thread[id=34785, name=zkCallback-1779-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=34783, name=zkCallback-1779-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([94457C292105B975]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=34696, name=zkCallback-1779-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=34694, name=StoppableCommitThread-SendThread(127.0.0.1:59721), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 3) Thread[id=34784, name=zkCallback-1779-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=34785, name=zkCallback-1779-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=34783, name=zkCallback-1779-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=34696, name=zkCallback-1779-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=34694, name=StoppableCommitThread-SendThread(127.0.0.1:59721), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
3) Thread[id=34784, name=zkCallback-1779-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=34785, name=zkCallback-1779-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=34783, name=zkCallback-1779-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([94457C292105B975]:0)
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([94457C292105B975:1C1143F38FF9D48D]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([94457C292105B975:1C1143F38FF9D48D]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:174)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([94457C292105B975:1C1143F38FF9D48D]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12023 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_94457C292105B975-001/init-core-data-001
[junit4] 2> 1196807 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[94457C292105B975]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 1196807 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[94457C292105B975]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 1196809 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[94457C292105B975]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 1196810 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[94457C292105B975]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 1196903 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[94457C292105B975]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 1196916 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[94457C292105B975]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1196919 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[94457C292105B975]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 1196935 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[94457C292105B975]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_33557_hdfs____.y5f4m7/webapp
[junit4] 2> 1197337 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[94457C292105B975]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33557
[junit4] 2> 1197449 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[94457C292105B975]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1197451 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[94457C292105B975]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 1197469 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[94457C292105B975]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_58569_datanode____.179so4/webapp
[junit4] 2> 1197941 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[94457C292105B975]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:58569
[junit4] 2> 1198288 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[94457C292105B975]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1198296 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[94457C292105B975]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 1198340 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[94457C292105B975]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_41784_datanode____lw59kt/webapp
[junit4] 2> 1198407 INFO (IPC Server handler 3 on 57857) [ ] BlockStateChange BLOCK* processReport: from storage DS-7c3697bf-910a-4aee-995f-47eea19e4338 node DatanodeRegistration(127.0.0.1:45398, datanodeUuid=214b7518-2641-44aa-9dff-50ba0277b609, infoPort=43353, infoSecurePort=0, ipcPort=46062, storageInfo=lv=-56;cid=testClusterID;nsid=404790946;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 1198407 INFO (IPC Server handler 3 on 57857) [ ] BlockStateChange BLOCK* processReport: from storage DS-e91d5731-1fbc-4b35-9bd4-de3e84255236 node DatanodeRegistration(127.0.0.1:45398, datanodeUuid=214b7518-2641-44aa-9dff-50ba0277b609, infoPort=43353, infoSecurePort=0, ipcPort=46062, storageInfo=lv=-56;cid=testClusterID;nsid=404790946;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 1198899 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[94457C292105B975]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41784
[junit4] 2> 1199291 INFO (IPC Server handler 5 on 57857) [ ] BlockStateChange BLOCK* processReport: from storage DS-b5aa13b9-6b18-46a5-ae8e-d2dbad4205ca node DatanodeRegistration(127.0.0.1:35631, datanodeUuid=85607d6f-d0f8-47fe-9d15-0b6b93cad27f, infoPort=48165, infoSecurePort=0, ipcPort=53057, storageInfo=lv=-56;cid=testClusterID;nsid=404790946;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 1199291 INFO (IPC Server handler 5 on 57857) [ ] BlockStateChange BLOCK* processReport: from storage DS-da72d026-7ed6-46a2-9b84-7fecc837d857 node DatanodeRegistration(127.0.0.1:35631, datanodeUuid=85607d6f-d0f8-47fe-9d15-0b6b93cad27f, infoPort=48165, infoSecurePort=0, ipcPort=53057, storageInfo=lv=-56;cid=testClusterID;nsid=404790946;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 1199882 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1199882 INFO (Thread-2325) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1199882 INFO (Thread-2325) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1199890 ERROR (Thread-2325) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1199982 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.ZkTestServer start zk server on port:60417
[junit4] 2> 1200076 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1200084 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 1200086 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1200087 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1200089 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1200090 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1200097 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1200098 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1200099 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1200101 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1200103 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1200108 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 1200314 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1200315 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2188ad94{/,null,AVAILABLE}
[junit4] 2> 1200316 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@44df461{SSL,[ssl, http/1.1]}{127.0.0.1:51626}
[junit4] 2> 1200316 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.e.j.s.Server Started @1204327ms
[junit4] 2> 1200316 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:57857/hdfs__localhost_57857__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.0_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_94457C292105B975-001_tempDir-002_control_data, replicaType=NRT, hostContext=/, hostPort=51626, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_94457C292105B975-001/control-001/cores}
[junit4] 2> 1200316 ERROR (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1200316 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1200316 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1200316 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1200317 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-09T03:06:28.174Z
[junit4] 2> 1200321 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1200321 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_94457C292105B975-001/control-001/solr.xml
[junit4] 2> 1200327 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@65e8387a, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1200331 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60417/solr
[junit4] 2> 1200418 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:51626_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1200420 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:51626_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:51626_
[junit4] 2> 1200421 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:51626_ ] o.a.s.c.Overseer Overseer (id=98451324124594181-127.0.0.1:51626_-n_0000000000) starting
[junit4] 2> 1200428 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:51626_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51626_
[junit4] 2> 1200433 INFO (zkCallback-1031-thread-1-processing-n:127.0.0.1:51626_) [n:127.0.0.1:51626_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1200505 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:51626_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@65e8387a
[junit4] 2> 1200511 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:51626_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@65e8387a
[junit4] 2> 1200512 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:51626_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@65e8387a
[junit4] 2> 1200513 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:51626_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_94457C292105B975-001/control-001/cores
[junit4] 2> 1200550 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1200551 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60417/solr ready
[junit4] 2> 1200599 INFO (qtp1787748001-7711) [n:127.0.0.1:51626_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:51626_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1200612 INFO (OverseerThreadFactory-3120-thread-1-processing-n:127.0.0.1:51626_) [n:127.0.0.1:51626_ ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 1200755 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1200756 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1200867 INFO (zkCallback-1031-thread-1-processing-n:127.0.0.1:51626_) [n:127.0.0.1:51626_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1201793 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1201810 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 1201921 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1201943 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 1201944 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@65e8387a
[junit4] 2> 1201944 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:57857/solr_hdfs_home
[junit4] 2> 1201944 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 1201944 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1201945 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_94457C292105B975-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://localhost:57857/solr_hdfs_home/control_collection/core_node1/data/]
[junit4] 2> 1201946 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:57857/solr_hdfs_home/control_collection/core_node1/data/snapshot_metadata
[junit4] 2> 1201955 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 1201955 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 1201963 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 1201964 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:57857/solr_hdfs_home/control_collection/core_node1/data
[junit4] 2> 1201983 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:57857/solr_hdfs_home/control_collection/core_node1/data/index
[junit4] 2> 1201990 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 1201990 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 1201994 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 1201995 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=2, maxMergeAtOnceExplicit=9, maxMergedSegmentMB=19.69921875, floorSegmentMB=0.9169921875, forceMergeDeletesPctAllowed=27.17299709751066, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1202033 INFO (IPC Server handler 8 on 57857) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:35631 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-e91d5731-1fbc-4b35-9bd4-de3e84255236:NORMAL:127.0.0.1:45398|RBW], ReplicaUC[[DISK]DS-b5aa13b9-6b18-46a5-ae8e-d2dbad4205ca:NORMAL:127.0.0.1:35631|FINALIZED]]} size 0
[junit4] 2> 1202036 INFO (IPC Server handler 0 on 57857) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:45398 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b5aa13b9-6b18-46a5-ae8e-d2dbad4205ca:NORMAL:127.0.0.1:35631|FINALIZED], ReplicaUC[[DISK]DS-7c3697bf-910a-4aee-995f-47eea19e4338:NORMAL:127.0.0.1:45398|FINALIZED]]} size 0
[junit4] 2> 1202043 WARN (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1202109 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 1202109 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1202109 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 1202124 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 1202125 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1202128 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=56.9345703125, floorSegmentMB=1.0283203125, forceMergeDeletesPctAllowed=14.224605154862793, segmentsPerTier=8.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39320151622720734
[junit4] 2> 1202147 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@712570fe[control_collection_shard1_replica_n1] main]
[junit4] 2> 1202148 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1202149 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1202150 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1202153 INFO (searcherExecutor-3123-thread-1-processing-n:127.0.0.1:51626_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@712570fe[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1202153 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1575221188373774336
[junit4] 2> 1202160 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1202160 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1202160 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:51626/control_collection_shard1_replica_n1/
[junit4] 2> 1202160 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1202160 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:51626/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 1202160 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1202165 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:51626/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 1202269 INFO (zkCallback-1031-thread-1-processing-n:127.0.0.1:51626_) [n:127.0.0.1:51626_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1202316 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1202320 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1565
[junit4] 2> 1202324 INFO (qtp1787748001-7711) [n:127.0.0.1:51626_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1202421 INFO (zkCallback-1031-thread-1-processing-n:127.0.0.1:51626_) [n:127.0.0.1:51626_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1202619 INFO (OverseerCollectionConfigSetProcessor-98451324124594181-127.0.0.1:51626_-n_0000000000) [n:127.0.0.1:51626_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1203324 INFO (qtp1787748001-7711) [n:127.0.0.1:51626_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:51626_&wt=javabin&version=2} status=0 QTime=2725
[junit4] 2> 1203366 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1203368 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60417/solr ready
[junit4] 2> 1203368 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1203371 INFO (qtp1787748001-7713) [n:127.0.0.1:51626_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1203374 INFO (OverseerThreadFactory-3120-thread-2-processing-n:127.0.0.1:51626_) [n:127.0.0.1:51626_ ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 1203375 WARN (OverseerThreadFactory-3120-thread-2-processing-n:127.0.0.1:51626_) [n:127.0.0.1:51626_ ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 1203584 INFO (qtp1787748001-7713) [n:127.0.0.1:51626_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1203584 INFO (qtp1787748001-7713) [n:127.0.0.1:51626_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=213
[junit4] 2> 1203763 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_94457C292105B975-001/shard-1-001 of type NRT
[junit4] 2> 1203765 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1203766 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1ee8d8f0{/,null,AVAILABLE}
[junit4] 2> 1203767 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1f6829fe{SSL,[ssl, http/1.1]}{127.0.0.1:35759}
[junit4] 2> 1203767 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.e.j.s.Server Started @1207778ms
[junit4] 2> 1203768 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:57857/hdfs__localhost_57857__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.0_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_94457C292105B975-001_tempDir-002_jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=35759, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_94457C292105B975-001/shard-1-001/cores}
[junit4] 2> 1203768 ERROR (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1203768 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1203768 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1203768 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1203768 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-09T03:06:31.626Z
[junit4] 2> 1203771 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1203772 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_94457C292105B975-001/shard-1-001/solr.xml
[junit4] 2> 1203777 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@65e8387a, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1203781 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60417/solr
[junit4] 2> 1203791 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:35759_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1203794 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:35759_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1203797 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:35759_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35759_
[junit4] 2> 1203799 INFO (zkCallback-1031-thread-1-processing-n:127.0.0.1:51626_) [n:127.0.0.1:51626_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1203799 INFO (zkCallback-1038-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1203801 INFO (zkCallback-1043-thread-1-processing-n:127.0.0.1:35759_) [n:127.0.0.1:35759_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1203887 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:35759_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@65e8387a
[junit4] 2> 1203894 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:35759_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@65e8387a
[junit4] 2> 1203894 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:35759_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@65e8387a
[junit4] 2> 1203896 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:35759_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_94457C292105B975-001/shard-1-001/cores
[junit4] 2> 1203924 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:35759_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1203927 INFO (OverseerCollectionConfigSetProcessor-98451324124594181-127.0.0.1:51626_-n_0000000000) [n:127.0.0.1:51626_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1203933 INFO (OverseerThreadFactory-3120-thread-3-processing-n:127.0.0.1:51626_) [n:127.0.0.1:51626_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:35759_ for creating new replica
[junit4] 2> 1203964 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1203965 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1204070 INFO (zkCallback-1043-thread-1-processing-n:127.0.0.1:35759_) [n:127.0.0.1:35759_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1204988 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1205003 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema [collection1_shard2_replica_n0] Schema name=test
[junit4] 2> 1205113 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1205130 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n0' using configuration from collection collection1, trusted=true
[junit4] 2> 1205131 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n0' (registry 'solr.core.collection1.shard2.replica_n0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@65e8387a
[junit4] 2> 1205131 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:57857/solr_hdfs_home
[junit4] 2> 1205131 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 1205131 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1205131 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore [[collection1_shard2_replica_n0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_94457C292105B975-001/shard-1-001/cores/collection1_shard2_replica_n0], dataDir=[hdfs://localhost:57857/solr_hdfs_home/collection1/core_node1/data/]
[junit4] 2> 1205133 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:57857/solr_hdfs_home/collection1/core_node1/data/snapshot_metadata
[junit4] 2> 1205141 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 1205141 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 1205145 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 1205146 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:57857/solr_hdfs_home/collection1/core_node1/data
[junit4] 2> 1205164 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:57857/solr_hdfs_home/collection1/core_node1/data/index
[junit4] 2> 1205171 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 1205171 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 1205175 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 1205176 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=2, maxMergeAtOnceExplicit=9, maxMergedSegmentMB=19.69921875, floorSegmentMB=0.9169921875, forceMergeDeletesPctAllowed=27.17299709751066, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1205191 INFO (IPC Server handler 7 on 57857) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:45398 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b5aa13b9-6b18-46a5-ae8e-d2dbad4205ca:NORMAL:127.0.0.1:35631|RBW], ReplicaUC[[DISK]DS-e91d5731-1fbc-4b35-9bd4-de3e84255236:NORMAL:127.0.0.1:45398|RBW]]} size 0
[junit4] 2> 1205192 INFO (IPC Server handler 9 on 57857) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:35631 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-e91d5731-1fbc-4b35-9bd4-de3e84255236:NORMAL:127.0.0.1:45398|RBW], ReplicaUC[[DISK]DS-da72d026-7ed6-46a2-9b84-7fecc837d857:NORMAL:127.0.0.1:35631|FINALIZED]]} size 0
[junit4] 2> 1205198 WARN (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1205253 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 1205253 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1205253 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 1205265 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 1205265 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1205267 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=56.9345703125, floorSegmentMB=1.0283203125, forceMergeDeletesPctAllowed=14.224605154862793, segmentsPerTier=8.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39320151622720734
[junit4] 2> 1205274 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.SolrIndexSearcher Opening [Searcher@f28d94f[collection1_shard2_replica_n0] main]
[junit4] 2> 1205275 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1205276 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1205276 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1205279 INFO (searcherExecutor-3134-thread-1-processing-n:127.0.0.1:35759_ x:collection1_shard2_replica_n0 s:shard2 c:collection1) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore [collection1_shard2_replica_n0] Registered new searcher Searcher@f28d94f[collection1_shard2_replica_n0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1205279 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1575221191651622912
[junit4] 2> 1205286 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1205286 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1205286 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:35759/collection1_shard2_replica_n0/
[junit4] 2> 1205286 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1205286 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy https://127.0.0.1:35759/collection1_shard2_replica_n0/ has no replicas
[junit4] 2> 1205286 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1205291 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:35759/collection1_shard2_replica_n0/ shard2
[junit4] 2> 1205394 INFO (zkCallback-1043-thread-1-processing-n:127.0.0.1:35759_) [n:127.0.0.1:35759_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1205442 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1205445 INFO (qtp1632195478-7762) [n:127.0.0.1:35759_ c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1480
[junit4] 2> 1205448 INFO (qtp1787748001-7706) [n:127.0.0.1:51626_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:35759_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1523
[junit4] 2> 1205547 INFO (zkCallback-1043-thread-1-processing-n:127.0.0.1:35759_) [n:127.0.0.1:35759_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1205601 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_94457C292105B975-001/shard-2-001 of type NRT
[junit4] 2> 1205603 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1205604 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@76512301{/,null,AVAILABLE}
[junit4] 2> 1205609 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3e8d5b55{SSL,[ssl, http/1.1]}{127.0.0.1:39435}
[junit4] 2> 1205609 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.e.j.s.Server Started @1209620ms
[junit4] 2> 1205609 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:57857/hdfs__localhost_57857__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.0_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_94457C292105B975-001_tempDir-002_jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=39435, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_94457C292105B975-001/shard-2-001/cores}
[junit4] 2> 1205609 ERROR (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1205610 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1205610 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1205610 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1205610 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-09T03:06:33.468Z
[junit4] 2> 1205614 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1205614 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_94457C292105B975-001/shard-2-001/solr.xml
[junit4] 2> 1205623 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@65e8387a, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1205628 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60417/solr
[junit4] 2> 1205639 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:39435_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1205642 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:39435_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1205645 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [n:127.0.0.1:39435_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39435_
[junit4] 2> 12
[...truncated too long message...]
92105B975]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@361bbe49{SSL,[ssl, http/1.1]}{127.0.0.1:0}
[junit4] 2> 2523633 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@69b3def1{/,null,UNAVAILABLE}
[junit4] 2> 2523634 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.ChaosMonkey monkey: stop jetty! 33236
[junit4] 2> 2523634 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=296579086
[junit4] 2> 2523634 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 2523634 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 2523634 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 2523635 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 2523637 WARN (zkCallback-3187-thread-4-processing-n:127.0.0.1:33236_) [n:127.0.0.1:33236_ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 2524610 WARN (zkCallback-3187-thread-2-processing-n:127.0.0.1:33236_) [n:127.0.0.1:33236_ c:collection1 s:shard3 r:core_node8 x:collection1_shard3_replica_n2] o.a.s.c.SyncStrategy Closed, skipping sync up.
[junit4] 2> 2524610 INFO (zkCallback-3187-thread-2-processing-n:127.0.0.1:33236_) [n:127.0.0.1:33236_ c:collection1 s:shard3 r:core_node8 x:collection1_shard3_replica_n2] o.a.s.c.SolrCore [collection1_shard3_replica_n2] CLOSING SolrCore org.apache.solr.core.SolrCore@43ab6305
[junit4] 2> 2524612 INFO (zkCallback-3187-thread-2-processing-n:127.0.0.1:33236_) [n:127.0.0.1:33236_ c:collection1 s:shard3 r:core_node8 x:collection1_shard3_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1.shard3.replica_n2, tag=1135305477
[junit4] 2> 2524616 INFO (zkCallback-3187-thread-2-processing-n:127.0.0.1:33236_) [n:127.0.0.1:33236_ c:collection1 s:shard3 r:core_node8 x:collection1_shard3_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard3.leader, tag=1135305477
[junit4] 2> 2524617 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@6f8eb489{SSL,[ssl, http/1.1]}{127.0.0.1:0}
[junit4] 2> 2524618 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@54387bf5{/,null,UNAVAILABLE}
[junit4] 2> 2524618 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2524620 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[94457C292105B975]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:57984 57984
[junit4] 2> 2524727 INFO (Thread-15168) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:57984 57984
[junit4] 2> 2524730 WARN (Thread-15168) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 11 /solr/aliases.json
[junit4] 2> 9 /solr/security.json
[junit4] 2> 9 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 11 /solr/clusterprops.json
[junit4] 2> 11 /solr/clusterstate.json
[junit4] 2> 8 /solr/collections/collection1/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 11 /solr/live_nodes
[junit4] 2> 11 /solr/collections
[junit4] 2> 5 /solr/overseer/collection-queue-work
[junit4] 2> 4 /solr/overseer/queue
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ChaosMonkeySafeLeaderTest -Dtests.method=test -Dtests.seed=94457C292105B975 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=sr-ME -Dtests.timezone=SystemV/EST5 -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] FAILURE 112s J2 | ChaosMonkeySafeLeaderTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
[junit4] > at __randomizedtesting.SeedInfo.seed([94457C292105B975:1C1143F38FF9D48D]:0)
[junit4] > at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
[junit4] > at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeySafeLeaderTest_94457C292105B975-001
[junit4] 2> Aug 09, 2017 3:28:32 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=HighCompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=17512, maxDocsPerChunk=598, blockSize=786), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=17512, blockSize=786)), sim=RandomSimilarity(queryNorm=true): {}, locale=sr-ME, timezone=SystemV/EST5
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=174978576,total=516947968
[junit4] 2> NOTE: All tests run in this JVM: [TestTolerantUpdateProcessorCloud, TestPullReplicaErrorHandling, EnumFieldTest, TestJmxIntegration, StressHdfsTest, TestImpersonationWithHadoopAuth, TestManagedStopFilterFactory, CurrencyFieldTypeTest, ShowFileRequestHandlerTest, TestDistributedMissingSort, HardAutoCommitTest, TestDistributedStatsComponentCardinality, AtomicUpdatesTest, SimplePostToolTest, MultiTermTest, CachingDirectoryFactoryTest, DirectUpdateHandlerOptimizeTest, DistributedFacetPivotSmallTest, DirectUpdateHandlerTest, TestOnReconnectListenerSupport, SolrXmlInZkTest, TestSubQueryTransformer, TestLegacyNumericRangeQueryBuilder, TestNestedDocsSort, TestSimpleTextCodec, AddBlockUpdateTest, TestStressUserVersions, CdcrReplicationHandlerTest, InfixSuggestersTest, HighlighterMaxOffsetTest, HdfsLockFactoryTest, FullSolrCloudDistribCmdsTest, TestConfigSetsAPIZkFailure, TestPivotHelperCode, SmileWriterTest, TestStressReorder, DateMathParserTest, TestManagedResource, SuggesterFSTTest, PreAnalyzedFieldTest, TestCollationFieldDocValues, RankQueryTest, TestNRTOpen, TestFieldResource, TestDFRSimilarityFactory, TestReversedWildcardFilterFactory, SignatureUpdateProcessorFactoryTest, PolyFieldTest, SortByFunctionTest, TestTrie, TestCollectionAPIs, TestLegacyField, TestQuerySenderListener, TestLeaderInitiatedRecoveryThread, SegmentsInfoRequestHandlerTest, TestShortCircuitedRequests, MetricUtilsTest, FastVectorHighlighterTest, RulesTest, TestCloudInspectUtil, TestGraphTermsQParserPlugin, TestMiniSolrCloudCluster, UniqFieldsUpdateProcessorFactoryTest, HdfsNNFailoverTest, StatsComponentTest, CleanupOldIndexTest, ClusterStateTest, DeleteReplicaTest, TestAnalyzeInfixSuggestions, ChaosMonkeyNothingIsSafeWithPullReplicasTest, TestCustomSort, URLClassifyProcessorTest, HLLSerializationTest, DocumentBuilderTest, CursorMarkTest, PathHierarchyTokenizerFactoryTest, TestSolrDeletionPolicy2, TestSolrCloudSnapshots, IgnoreCommitOptimizeUpdateProcessorFactoryTest, PrimitiveFieldTypeTest, HdfsTlogReplayBufferedWhileIndexingTest, TestExportWriter, CollectionStateFormat2Test, TestDynamicFieldCollectionResource, TestCorePropertiesReload, TestLegacyTerms, TestRequestStatusCollectionAPI, CoreAdminRequestStatusTest, V2ApiIntegrationTest, TestSurroundQueryParser, ClassificationUpdateProcessorTest, ConnectionManagerTest, SpatialHeatmapFacetsTest, TestHdfsBackupRestoreCore, TestMaxTokenLenTokenizer, TestUtils, TestLRUStatsCache, QueryEqualityTest, TestConfigSetImmutable, MinimalSchemaTest, ResponseHeaderTest, TestSchemaNameResource, TestStressInPlaceUpdates, HdfsRestartWhileUpdatingTest, TestCollectionAPI, BasicDistributedZk2Test, TestCloudPseudoReturnFields, TestSolrConfigHandlerCloud, UninvertDocValuesMergePolicyTest, TestPayloadCheckQParserPlugin, UnloadDistributedZkTest, TestFoldingMultitermQuery, TestSolrCloudWithKerberosAlt, ForceLeaderTest, NumericFieldsTest, LeaderFailoverAfterPartitionTest, BaseCdcrDistributedZkTest, ChaosMonkeySafeLeaderTest]
[junit4] Completed [492/727 (3!)] on J2 in 112.82s, 1 test, 1 failure <<< FAILURES!
[...truncated 44028 lines...]
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 32 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/32/
No tests ran.
Build Log:
[...truncated 2079 lines...]
ERROR: command execution failed.
ERROR: Step ‘Archive the artifacts’ failed: no workspace for Lucene-Solr-NightlyTests-7.0 #32
ERROR: Step ‘Publish JUnit test result report’ failed: no workspace for Lucene-Solr-NightlyTests-7.0 #32
ERROR: lucene2 is offline; cannot locate JDK 1.8 (latest)
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
ERROR: lucene2 is offline; cannot locate JDK 1.8 (latest)
ERROR: lucene2 is offline; cannot locate JDK 1.8 (latest)
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 31 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/31/
3 tests failed.
FAILED: org.apache.solr.core.HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig
Error Message:
The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~134.217728 MB.
Stack Trace:
java.lang.RuntimeException: The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~134.217728 MB.
at __randomizedtesting.SeedInfo.seed([D521B1D1DF7A7797:228E78FA22F39DBC]:0)
at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:311)
at org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:287)
at org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:227)
at org.apache.solr.core.HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig(HdfsDirectoryFactoryTest.java:111)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.OutOfMemoryError: Direct buffer memory
at java.nio.Bits.reserveMemory(Bits.java:694)
at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)
at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)
at org.apache.solr.store.blockcache.BlockCache.<init>(BlockCache.java:71)
at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:309)
... 42 more
FAILED: org.apache.solr.cloud.HttpPartitionTest.test
Error Message:
Doc with id=954 not found in http://127.0.0.1:58914/c8n_1x2 due to: Path not found: /id; rsp={doc=null}
Stack Trace:
java.lang.AssertionError: Doc with id=954 not found in http://127.0.0.1:58914/c8n_1x2 due to: Path not found: /id; rsp={doc=null}
at __randomizedtesting.SeedInfo.seed([D521B1D1DF7A7797:5D758E0B71861A6F]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.HttpPartitionTest.assertDocExists(HttpPartitionTest.java:603)
at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:558)
at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:379)
at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
Timeout waiting for all live and active
Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
at __randomizedtesting.SeedInfo.seed([D521B1D1DF7A7797:5D758E0B71861A6F]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:317)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11722 lines...]
[junit4] Suite: org.apache.solr.core.HdfsDirectoryFactoryTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_D521B1D1DF7A7797-001/init-core-data-001
[junit4] 2> 955223 WARN (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 955229 INFO (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 955230 INFO (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 955278 WARN (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 955296 WARN (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 955298 INFO (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 955320 INFO (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_45311_hdfs____3yn8yk/webapp
[junit4] 2> 955900 INFO (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45311
[junit4] 2> 956005 WARN (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 956006 INFO (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 956014 INFO (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_56633_datanode____ns6r0c/webapp
[junit4] 2> 956552 INFO (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:56633
[junit4] 2> 956627 WARN (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 956628 INFO (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 956639 INFO (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_40006_datanode____.fz2wv3/webapp
[junit4] 2> 956701 INFO (IPC Server handler 3 on 33295) [ ] BlockStateChange BLOCK* processReport: from storage DS-e63e7b84-5d60-4dbf-9c31-930416fbeaee node DatanodeRegistration(127.0.0.1:55569, datanodeUuid=f9f2727c-49ca-48a5-82db-0c75c87a5596, infoPort=51754, infoSecurePort=0, ipcPort=44754, storageInfo=lv=-56;cid=testClusterID;nsid=178299505;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 956701 INFO (IPC Server handler 3 on 33295) [ ] BlockStateChange BLOCK* processReport: from storage DS-5886711b-1313-4e98-96d9-2a1b8738828a node DatanodeRegistration(127.0.0.1:55569, datanodeUuid=f9f2727c-49ca-48a5-82db-0c75c87a5596, infoPort=51754, infoSecurePort=0, ipcPort=44754, storageInfo=lv=-56;cid=testClusterID;nsid=178299505;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 957272 INFO (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40006
[junit4] 2> 957429 INFO (IPC Server handler 4 on 33295) [ ] BlockStateChange BLOCK* processReport: from storage DS-1ba8a2f6-a7f8-41e6-aac8-3efcc265f79f node DatanodeRegistration(127.0.0.1:34888, datanodeUuid=e50d7cb2-8b20-41ce-9d4c-2aad3cba5cb2, infoPort=45971, infoSecurePort=0, ipcPort=39635, storageInfo=lv=-56;cid=testClusterID;nsid=178299505;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs
[junit4] 2> 957429 INFO (IPC Server handler 4 on 33295) [ ] BlockStateChange BLOCK* processReport: from storage DS-52a6150b-e1b3-46b5-ba6c-5d43ee26053f node DatanodeRegistration(127.0.0.1:34888, datanodeUuid=e50d7cb2-8b20-41ce-9d4c-2aad3cba5cb2, infoPort=45971, infoSecurePort=0, ipcPort=39635, storageInfo=lv=-56;cid=testClusterID;nsid=178299505;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 957491 INFO (TEST-HdfsDirectoryFactoryTest.testCleanupOldIndexDirectories-seed#[D521B1D1DF7A7797]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testCleanupOldIndexDirectories
[junit4] 2> 957492 INFO (TEST-HdfsDirectoryFactoryTest.testCleanupOldIndexDirectories-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:33295/solr1
[junit4] 2> 957492 INFO (TEST-HdfsDirectoryFactoryTest.testCleanupOldIndexDirectories-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 957526 INFO (TEST-HdfsDirectoryFactoryTest.testCleanupOldIndexDirectories-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.HdfsDirectoryFactory Found 1 old index directories to clean-up under hdfs://localhost:33295/solr1/mock/data afterReload=false
[junit4] 2> 957529 INFO (TEST-HdfsDirectoryFactoryTest.testCleanupOldIndexDirectories-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.HdfsDirectoryFactory Deleted old index directory hdfs://localhost:33295/solr1/mock/data/index.20170819084910190
[junit4] 2> 957529 INFO (TEST-HdfsDirectoryFactoryTest.testCleanupOldIndexDirectories-seed#[D521B1D1DF7A7797]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testCleanupOldIndexDirectories
[junit4] 2> 957576 INFO (TEST-HdfsDirectoryFactoryTest.testLocalityReporter-seed#[D521B1D1DF7A7797]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testLocalityReporter
[junit4] 2> 957581 INFO (TEST-HdfsDirectoryFactoryTest.testLocalityReporter-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:33295/solr
[junit4] 2> 957581 INFO (TEST-HdfsDirectoryFactoryTest.testLocalityReporter-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 957581 INFO (TEST-HdfsDirectoryFactoryTest.testLocalityReporter-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:33295/solr3/
[junit4] 2> 957588 INFO (TEST-HdfsDirectoryFactoryTest.testLocalityReporter-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.h.HdfsLocalityReporter Registering direcotry hdfs://localhost:33295/solr3 for locality metrics.
[junit4] 2> 957618 INFO (IPC Server handler 7 on 33295) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34888 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-e63e7b84-5d60-4dbf-9c31-930416fbeaee:NORMAL:127.0.0.1:55569|RBW], ReplicaUC[[DISK]DS-52a6150b-e1b3-46b5-ba6c-5d43ee26053f:NORMAL:127.0.0.1:34888|FINALIZED]]} size 0
[junit4] 2> 957633 INFO (IPC Server handler 8 on 33295) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:55569 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-e63e7b84-5d60-4dbf-9c31-930416fbeaee:NORMAL:127.0.0.1:55569|RBW], ReplicaUC[[DISK]DS-52a6150b-e1b3-46b5-ba6c-5d43ee26053f:NORMAL:127.0.0.1:34888|FINALIZED]]} size 0
[junit4] 2> 957641 INFO (TEST-HdfsDirectoryFactoryTest.testLocalityReporter-seed#[D521B1D1DF7A7797]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testLocalityReporter
[junit4] 2> 957745 INFO (TEST-HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig-seed#[D521B1D1DF7A7797]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testInitArgsOrSysPropConfig
[junit4] 2> 957745 INFO (TEST-HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:33295/solr1
[junit4] 2> 957745 INFO (TEST-HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 957746 INFO (TEST-HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:33295/solr2
[junit4] 2> 957746 INFO (TEST-HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 957746 INFO (TEST-HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:33295/solr2
[junit4] 2> 957746 INFO (TEST-HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 957746 INFO (TEST-HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:33295/solr
[junit4] 2> 957753 INFO (TEST-HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 957753 INFO (TEST-HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [134217728] will allocate [1] slabs and use ~[134217728] bytes
[junit4] 2> 957753 INFO (TEST-HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 958808 ERROR (Finalizer) [ ] o.a.s.c.CoreContainer CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!! instance=547142799
[junit4] 2> 959310 INFO (TEST-HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig-seed#[D521B1D1DF7A7797]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testInitArgsOrSysPropConfig
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=HdfsDirectoryFactoryTest -Dtests.method=testInitArgsOrSysPropConfig -Dtests.seed=D521B1D1DF7A7797 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=es -Dtests.timezone=Atlantic/Azores -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 1.68s J1 | HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig <<<
[junit4] > Throwable #1: java.lang.RuntimeException: The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~134.217728 MB.
[junit4] > at __randomizedtesting.SeedInfo.seed([D521B1D1DF7A7797:228E78FA22F39DBC]:0)
[junit4] > at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:311)
[junit4] > at org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:287)
[junit4] > at org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:227)
[junit4] > at org.apache.solr.core.HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig(HdfsDirectoryFactoryTest.java:111)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > Caused by: java.lang.OutOfMemoryError: Direct buffer memory
[junit4] > at java.nio.Bits.reserveMemory(Bits.java:694)
[junit4] > at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)
[junit4] > at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)
[junit4] > at org.apache.solr.store.blockcache.BlockCache.<init>(BlockCache.java:71)
[junit4] > at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:309)
[junit4] > ... 42 more
[junit4] 2> 959320 WARN (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 959335 INFO (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 959437 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_D521B1D1DF7A7797-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_D521B1D1DF7A7797-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:33295) [ ] o.a.h.h.s.d.DataNode BPOfferService for Block pool BP-182147182-10.41.0.5-1503132547946 (Datanode Uuid e50d7cb2-8b20-41ce-9d4c-2aad3cba5cb2) service to localhost/127.0.0.1:33295 interrupted
[junit4] 2> 959438 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_D521B1D1DF7A7797-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_D521B1D1DF7A7797-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:33295) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-182147182-10.41.0.5-1503132547946 (Datanode Uuid e50d7cb2-8b20-41ce-9d4c-2aad3cba5cb2) service to localhost/127.0.0.1:33295
[junit4] 2> 959442 WARN (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 959464 INFO (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 959465 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_D521B1D1DF7A7797-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_D521B1D1DF7A7797-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:33295) [ ] o.a.h.h.s.d.DataNode BPOfferService for Block pool BP-182147182-10.41.0.5-1503132547946 (Datanode Uuid f9f2727c-49ca-48a5-82db-0c75c87a5596) service to localhost/127.0.0.1:33295 interrupted
[junit4] 2> 959466 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_D521B1D1DF7A7797-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_D521B1D1DF7A7797-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:33295) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-182147182-10.41.0.5-1503132547946 (Datanode Uuid f9f2727c-49ca-48a5-82db-0c75c87a5596) service to localhost/127.0.0.1:33295
[junit4] 2> 959483 INFO (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 959591 ERROR (SUITE-HdfsDirectoryFactoryTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.a.h.m.l.MethodMetric Error invoking method getBlocksTotal
[junit4] 2> java.lang.reflect.InvocationTargetException
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MethodMetric$2.snapshot(MethodMetric.java:111)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MethodMetric.snapshot(MethodMetric.java:144)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MetricsRegistry.snapshot(MetricsRegistry.java:401)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MetricsSourceBuilder$1.getMetrics(MetricsSourceBuilder.java:79)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
[junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
[junit4] 2> at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:204)
[junit4] 2> at org.apache.solr.core.HdfsDirectoryFactoryTest.teardownClass(HdfsDirectoryFactoryTest.java:64)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_D521B1D1DF7A7797-001
[junit4] 2> Aug 19, 2017 8:49:12 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=731, maxMBSortInHeap=6.046733668617431, sim=RandomSimilarity(queryNorm=false): {}, locale=es, timezone=Atlantic/Azores
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=280971696,total=512753664
[junit4] 2> NOTE: All tests run in this JVM: [TestElisionMultitermQuery, TestDocBasedVersionConstraints, RuleEngineTest, TestSegmentSorting, GraphQueryTest, BlockJoinFacetDistribTest, SuggesterTSTTest, TestSubQueryTransformerCrossCore, TestStressCloudBlindAtomicUpdates, TestSha256AuthenticationProvider, BlockDirectoryTest, TestJsonFacetRefinement, HdfsLockFactoryTest, TestCloudPivotFacet, TestStressVersions, TestClusterProperties, RemoteQueryErrorTest, DeleteStatusTest, TestSolrCloudWithDelegationTokens, SolrCoreMetricManagerTest, OutOfBoxZkACLAndCredentialsProvidersTest, HdfsBasicDistributedZkTest, ShardRoutingTest, SpatialFilterTest, TestManagedSchemaThreadSafety, ManagedSchemaRoundRobinCloudTest, IndexSchemaTest, InfoHandlerTest, TestFreeTextSuggestions, DistributedFacetExistsSmallTest, PrimUtilsTest, DocValuesMultiTest, ZkControllerTest, JavabinLoaderTest, TestClassNameShortening, TestFieldTypeCollectionResource, HdfsDirectoryFactoryTest]
[junit4] Completed [104/729 (1!)] on J1 in 14.18s, 3 tests, 1 error <<< FAILURES!
[...truncated 1410 lines...]
[junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/init-core-data-001
[junit4] 2> 3938487 WARN (SUITE-HttpPartitionTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=101 numCloses=101
[junit4] 2> 3938487 INFO (SUITE-HttpPartitionTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 3938488 INFO (SUITE-HttpPartitionTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 3938488 INFO (SUITE-HttpPartitionTest-seed#[D521B1D1DF7A7797]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 3938490 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3938490 INFO (Thread-4287) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3938490 INFO (Thread-4287) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3938491 ERROR (Thread-4287) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3938590 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.ZkTestServer start zk server on port:48085
[junit4] 2> 3938608 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 3938609 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 3938611 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 3938612 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 3938612 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 3938613 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 3938614 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 3938616 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 3938617 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 3938618 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 3938619 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 3938621 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
[junit4] 2> 3938728 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3938733 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@450f74be{/,null,AVAILABLE}
[junit4] 2> 3938733 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@569b25d1{HTTP/1.1,[http/1.1]}{127.0.0.1:43908}
[junit4] 2> 3938733 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.e.j.s.Server Started @3942930ms
[junit4] 2> 3938733 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/tempDir-001/control/data, hostContext=/, hostPort=58914, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/control-001/cores}
[junit4] 2> 3938733 ERROR (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3938733 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 3938733 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3938733 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 3938733 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-19T09:38:50.600Z
[junit4] 2> 3938742 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3938742 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/control-001/solr.xml
[junit4] 2> 3938746 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@47430182, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3938748 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48085/solr
[junit4] 2> 3938789 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:58914_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3938789 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:58914_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:58914_
[junit4] 2> 3938789 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:58914_ ] o.a.s.c.Overseer Overseer (id=98509490117541892-127.0.0.1:58914_-n_0000000000) starting
[junit4] 2> 3938801 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:58914_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58914_
[junit4] 2> 3938802 INFO (zkCallback-27415-thread-1-processing-n:127.0.0.1:58914_) [n:127.0.0.1:58914_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3938852 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:58914_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47430182
[junit4] 2> 3938860 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:58914_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47430182
[junit4] 2> 3938860 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:58914_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47430182
[junit4] 2> 3938861 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:58914_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/control-001/cores
[junit4] 2> 3938891 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3938892 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48085/solr ready
[junit4] 2> 3938892 INFO (SocketProxy-Acceptor-58914) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=46145,localport=58914], receiveBufferSize:531000
[junit4] 2> 3938904 INFO (SocketProxy-Acceptor-58914) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=43908,localport=34909], receiveBufferSize=530904
[junit4] 2> 3938911 INFO (qtp1975138459-113986) [n:127.0.0.1:58914_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:58914_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3938913 INFO (OverseerThreadFactory-10758-thread-1-processing-n:127.0.0.1:58914_) [n:127.0.0.1:58914_ ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 3939018 INFO (SocketProxy-Acceptor-58914) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=46151,localport=58914], receiveBufferSize:531000
[junit4] 2> 3939019 INFO (SocketProxy-Acceptor-58914) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=43908,localport=34915], receiveBufferSize=530904
[junit4] 2> 3939019 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3939021 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3939126 INFO (zkCallback-27415-thread-1-processing-n:127.0.0.1:58914_) [n:127.0.0.1:58914_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3940037 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 3940050 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 3940132 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3940141 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 3940142 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47430182
[junit4] 2> 3940142 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3940142 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 3940144 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=46, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.324891401515533]
[junit4] 2> 3940147 WARN (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3940218 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3940223 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3940226 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3940226 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3940227 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@4cea1ec7
[junit4] 2> 3940232 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@47bcc037[control_collection_shard1_replica_n1] main]
[junit4] 2> 3940233 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3940233 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3940234 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 3940234 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576151843708338176
[junit4] 2> 3940238 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3940238 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3940239 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:58914/control_collection_shard1_replica_n1/
[junit4] 2> 3940239 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3940239 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:58914/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 3940239 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3940241 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:58914/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 3940242 INFO (searcherExecutor-10761-thread-1-processing-n:127.0.0.1:58914_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@47bcc037[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3940342 INFO (zkCallback-27415-thread-1-processing-n:127.0.0.1:58914_) [n:127.0.0.1:58914_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3940392 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3940393 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1374
[junit4] 2> 3940395 INFO (qtp1975138459-113986) [n:127.0.0.1:58914_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3940495 INFO (zkCallback-27415-thread-2-processing-n:127.0.0.1:58914_) [n:127.0.0.1:58914_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3940914 INFO (OverseerCollectionConfigSetProcessor-98509490117541892-127.0.0.1:58914_-n_0000000000) [n:127.0.0.1:58914_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3941395 INFO (qtp1975138459-113986) [n:127.0.0.1:58914_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:58914_&wt=javabin&version=2} status=0 QTime=2484
[junit4] 2> 3941401 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3941402 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48085/solr ready
[junit4] 2> 3941402 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 3941402 INFO (SocketProxy-Acceptor-58914) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=46163,localport=58914], receiveBufferSize:531000
[junit4] 2> 3941403 INFO (qtp1975138459-113987) [n:127.0.0.1:58914_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3941412 INFO (SocketProxy-Acceptor-58914) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=43908,localport=34927], receiveBufferSize=530904
[junit4] 2> 3941416 INFO (OverseerThreadFactory-10758-thread-2-processing-n:127.0.0.1:58914_) [n:127.0.0.1:58914_ ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 3941416 WARN (OverseerThreadFactory-10758-thread-2-processing-n:127.0.0.1:58914_) [n:127.0.0.1:58914_ ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 3941626 INFO (qtp1975138459-113987) [n:127.0.0.1:58914_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3941626 INFO (qtp1975138459-113987) [n:127.0.0.1:58914_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=223
[junit4] 2> 3941720 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/shard-1-001 of type TLOG
[junit4] 2> 3941721 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3941722 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@45ce716e{/,null,AVAILABLE}
[junit4] 2> 3941722 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6c9ecb33{HTTP/1.1,[http/1.1]}{127.0.0.1:45735}
[junit4] 2> 3941722 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.e.j.s.Server Started @3945919ms
[junit4] 2> 3941722 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=59479, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/shard-1-001/cores}
[junit4] 2> 3941722 ERROR (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3941722 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 3941723 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3941723 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 3941723 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-19T09:38:53.590Z
[junit4] 2> 3941730 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3941730 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/shard-1-001/solr.xml
[junit4] 2> 3941749 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@47430182, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3941751 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48085/solr
[junit4] 2> 3941771 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:59479_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3941773 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:59479_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3941774 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:59479_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59479_
[junit4] 2> 3941775 INFO (zkCallback-27422-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3941775 INFO (zkCallback-27415-thread-2-processing-n:127.0.0.1:58914_) [n:127.0.0.1:58914_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3941781 INFO (zkCallback-27427-thread-1-processing-n:127.0.0.1:59479_) [n:127.0.0.1:59479_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3941847 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:59479_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47430182
[junit4] 2> 3941858 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:59479_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47430182
[junit4] 2> 3941858 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:59479_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47430182
[junit4] 2> 3941859 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:59479_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/shard-1-001/cores
[junit4] 2> 3941882 INFO (qtp1975138459-113988) [n:127.0.0.1:58914_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:59479_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3941884 INFO (OverseerCollectionConfigSetProcessor-98509490117541892-127.0.0.1:58914_-n_0000000000) [n:127.0.0.1:58914_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3941885 INFO (OverseerThreadFactory-10758-thread-3-processing-n:127.0.0.1:58914_) [n:127.0.0.1:58914_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:59479_ for creating new replica
[junit4] 2> 3941886 INFO (SocketProxy-Acceptor-59479) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=44737,localport=59479], receiveBufferSize:531000
[junit4] 2> 3941887 INFO (SocketProxy-Acceptor-59479) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=45735,localport=40508], receiveBufferSize=530904
[junit4] 2> 3941896 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 3941896 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3942910 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 3942921 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.s.IndexSchema [collection1_shard2_replica_t0] Schema name=test
[junit4] 2> 3943013 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3943023 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t0' using configuration from collection collection1, trusted=true
[junit4] 2> 3943024 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t0' (registry 'solr.core.collection1.shard2.replica_t0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47430182
[junit4] 2> 3943024 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3943024 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SolrCore [[collection1_shard2_replica_t0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/shard-1-001/cores/collection1_shard2_replica_t0], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/shard-1-001/cores/collection1_shard2_replica_t0/data/]
[junit4] 2> 3943026 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=46, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.324891401515533]
[junit4] 2> 3943028 WARN (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3943060 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3943060 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3943061 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3943061 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3943062 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@4ae27b25
[junit4] 2> 3943062 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.s.SolrIndexSearcher Opening [Searcher@825745d[collection1_shard2_replica_t0] main]
[junit4] 2> 3943063 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3943063 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3943064 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 3943064 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576151846675808256
[junit4] 2> 3943069 INFO (searcherExecutor-10772-thread-1-processing-n:127.0.0.1:59479_ x:collection1_shard2_replica_t0 s:shard2 c:collection1) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SolrCore [collection1_shard2_replica_t0] Registered new searcher Searcher@825745d[collection1_shard2_replica_t0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3943069 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3943069 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3943069 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:59479/collection1_shard2_replica_t0/
[junit4] 2> 3943069 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3943069 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SyncStrategy http://127.0.0.1:59479/collection1_shard2_replica_t0/ has no replicas
[junit4] 2> 3943069 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3943069 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ZkController collection1_shard2_replica_t0 stopping background replication from leader
[junit4] 2> 3943073 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:59479/collection1_shard2_replica_t0/ shard2
[junit4] 2> 3943223 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3943225 INFO (qtp1891185082-114035) [n:127.0.0.1:59479_ c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1329
[junit4] 2> 3943228 INFO (qtp1975138459-113988) [n:127.0.0.1:58914_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:59479_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=1345
[junit4] 2> 3943340 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/shard-2-001 of type TLOG
[junit4] 2> 3943340 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3943341 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1a548eae{/,null,AVAILABLE}
[junit4] 2> 3943341 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@41898250{HTTP/1.1,[http/1.1]}{127.0.0.1:46546}
[junit4] 2> 3943341 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.e.j.s.Server Started @3947538ms
[junit4] 2> 3943341 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=41717, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/shard-2-001/cores}
[junit4] 2> 3943342 ERROR (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3943348 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 3943348 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3943348 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 3943348 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-19T09:38:55.215Z
[junit4] 2> 3943355 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3943355 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/shard-2-001/solr.xml
[junit4] 2> 3943361 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@47430182, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3943365 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48085/solr
[junit4] 2> 3943374 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:41717_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3943375 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:41717_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3943376 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:41717_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41717_
[junit4] 2> 3943377 INFO (zkCallback-27422-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3943378 INFO (zkCallback-27427-thread-1-processing-n:127.0.0.1:59479_) [n:127.0.0.1:59479_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3943379 INFO (zkCallback-27415-thread-2-processing-n:127.0.0.1:58914_) [n:127.0.0.1:58914_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3943382 INFO (zkCallback-27433-thread-1-processing-n:127.0.0.1:41717_) [n:127.0.0.1:41717_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3943429 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:41717_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47430182
[junit4] 2> 3943438 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:41717_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47430182
[junit4] 2> 3943438 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:41717_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47430182
[junit4] 2> 3943439 INFO (TEST-HttpPartitionTest.test-seed#[D521B1D1DF7A7797]) [n:127.0.0.1:41717_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/shard-2-001/cores
[junit4] 2> 3943472 INFO (qtp1975138459-113984) [n:127.0.0.1:58914_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41717_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3943478 INFO (OverseerCollectionConfigSetProcessor-98509490117541892-127.0.0.1:58914_-n_0000000000) [n:127.0.0.1:58914_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3943479 INFO (OverseerThreadFactory-10758-thread-4-processing-n:127.0.0.1:58914_) [n:127.0.0.1:58914_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:41717_ for creating new replica
[junit4] 2> 3943480 INFO (SocketProxy-Acceptor-41717) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=33046,localport=41717], receiveBufferSize:531000
[junit4] 2> 3943481 INFO (SocketProxy-Acceptor-41717) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46546,localport=40664], receiveBufferSize=530904
[junit4] 2> 3943485 INFO (qtp573672856-114067) [n:127.0.0.1:41717_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t0&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 3943486 INFO (qtp573672856-114067) [n:127.0.0.1:41717_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3944504 INFO (qtp573672856-114067) [n:127.0.0.1:41717_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 3944538 INFO (qtp573672856-114067) [n:127.0.0.1:41717_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.s.IndexSchema [collection1_shard1_replica_t0] Schema name=test
[junit4] 2> 3944634 INFO (qtp573672856-114067) [n:127.0.0.1:41717_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3944647 INFO (qtp573672856-114067) [n:127.0.0.1:41717_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t0' using configuration from collection collection1, trusted=true
[junit4] 2> 3944648 INFO (qtp573672856-114067) [n:127.0.0.1:41717_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t0' (registry 'solr.core.collection1.shard1.replica_t0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@47430182
[junit4] 2> 3944648 INFO (qtp573672856-114067) [n:127.0.0.1:41717_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3944648 INFO (qtp573672856-114067) [n:127.0.0.1:41717_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SolrCore [[collection1_shard1_replica_t0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/shard-2-001/cores/collection1_shard1_replica_t0], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_D521B1D1DF7A7797-001/shard-2-001/cores/collection1_shard1_replica_t0/data/]
[junit4] 2> 3944650 INFO (qtp573672856-114067) [n:127.0.0.1:41717_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByt
[...truncated too long message...]
org.apache.hadoop.metrics2.lib.MethodMetric$2.snapshot(MethodMetric.java:111)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MethodMetric.snapshot(MethodMetric.java:144)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MetricsRegistry.snapshot(MetricsRegistry.java:401)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MetricsSourceBuilder$1.getMetrics(MetricsSourceBuilder.java:79)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
[junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
[junit4] 2> at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:204)
[junit4] 2> at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.hdfsFailoverAfterClass(SharedFSAutoReplicaFailoverTest.java:97)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_D521B1D1DF7A7797-001
[junit4] 2> Aug 19, 2017 9:47:15 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 35 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=true): {}, locale=es-NI, timezone=Europe/Samara
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=69454112,total=521666560
[junit4] 2> NOTE: All tests run in this JVM: [StatsComponentTest, TestCollectionAPI, TestOnReconnectListenerSupport, ShardSplitTest, PreAnalyzedFieldManagedSchemaCloudTest, SortByFunctionTest, TestReversedWildcardFilterFactory, TestPolicyCloud, HdfsTlogReplayBufferedWhileIndexingTest, AliasIntegrationTest, TestDefaultStatsCache, TestNumericRangeQuery64, TestExactStatsCache, ParsingFieldUpdateProcessorsTest, TestJmxIntegration, TermsComponentTest, AnalyticsQueryTest, ExplicitHLLTest, OverseerRolesTest, BigEndianAscendingWordDeserializerTest, TestPseudoReturnFields, DeleteReplicaTest, TestLegacyField, RollingRestartTest, TestFieldTypeResource, ConnectionManagerTest, ChangedSchemaMergeTest, TestCloudPseudoReturnFields, V2ApiIntegrationTest, TestFieldCacheWithThreads, TestDFRSimilarityFactory, TestConfigSetsAPIZkFailure, RequiredFieldsTest, ReturnFieldsTest, TestRebalanceLeaders, TestBlendedInfixSuggestions, FullSolrCloudDistribCmdsTest, CleanupOldIndexTest, SignatureUpdateProcessorFactoryTest, LeaderElectionTest, TestSchemaSimilarityResource, HdfsNNFailoverTest, JvmMetricsTest, DeleteInactiveReplicaTest, BasicAuthStandaloneTest, MoveReplicaHDFSTest, TestSort, TestZkChroot, DocumentBuilderTest, SearchHandlerTest, DistribCursorPagingTest, PrimitiveFieldTypeTest, HighlighterMaxOffsetTest, UnloadDistributedZkTest, TestDistributedMissingSort, MultiThreadedOCPTest, TestLMJelinekMercerSimilarityFactory, SolrCoreCheckLockOnStartupTest, TestSolrCloudSnapshots, ShowFileRequestHandlerTest, TestManagedSynonymGraphFilterFactory, TestTrie, TestSearcherReuse, SpatialHeatmapFacetsTest, HardAutoCommitTest, TestLeaderInitiatedRecoveryThread, SimpleCollectionCreateDeleteTest, TestCollectionAPIs, MultiTermTest, RulesTest, UniqFieldsUpdateProcessorFactoryTest, SimpleFacetsTest, TestFieldCacheSort, TestPivotHelperCode, TestDelegationWithHadoopAuth, ClusterStateTest, BufferStoreTest, TestRestManager, DateFieldTest, TestJsonFacets, OverriddenZkACLAndCredentialsProvidersTest, TestPKIAuthenticationPlugin, RecoveryZkTest, CdcrReplicationHandlerTest, TestOverriddenPrefixQueryForCustomFieldType, PeerSyncWithIndexFingerprintCachingTest, SolrShardReporterTest, SpellPossibilityIteratorTest, BasicDistributedZk2Test, TestRestoreCore, TestPointFields, TestSolrConfigHandlerCloud, TestHighFrequencyDictionaryFactory, TestIndexSearcher, TestFieldCollectionResource, TestSuggestSpellingConverter, TestQueryUtils, MetricsHandlerTest, TestAnalyzedSuggestions, TestStressInPlaceUpdates, TermVectorComponentDistributedTest, InfixSuggestersTest, TestShortCircuitedRequests, TestIntervalFaceting, TestDistribDocBasedVersion, AnalysisErrorHandlingTest, TestCloudNestedDocsSort, DistributedQueryComponentOptimizationTest, TestRecovery, TestGraphMLResponseWriter, QueryEqualityTest, TestFoldingMultitermQuery, LeaderFailoverAfterPartitionTest, JSONWriterTest, SolrGangliaReporterTest, CoreAdminRequestStatusTest, MinimalSchemaTest, SolrXmlInZkTest, HdfsRestartWhileUpdatingTest, NodeMutatorTest, OverseerModifyCollectionTest, OverseerStatusTest, PeerSyncReplicationTest, ReplaceNodeTest, SSLMigrationTest, SaslZkACLProviderTest, SharedFSAutoReplicaFailoverTest]
[junit4] Completed [548/729 (3!)] on J2 in 307.77s, 1 test, 1 failure <<< FAILURES!
[...truncated 43898 lines...]
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 30 - Still
unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/30/
6 tests failed.
FAILED: org.apache.lucene.index.TestBinaryDocValuesUpdates.testTonsOfUpdates
Error Message:
_2_4.fnm in dir=RAMDirectory@614bdb2f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@448315ab
Stack Trace:
java.io.FileNotFoundException: _2_4.fnm in dir=RAMDirectory@614bdb2f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@448315ab
at __randomizedtesting.SeedInfo.seed([A3EFB8732F26DD8:721B258CD0D2423A]:0)
at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:750)
at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:119)
at org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1072)
at org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat.read(Lucene60FieldInfosFormat.java:113)
at org.apache.lucene.index.SegmentReader.initFieldInfos(SegmentReader.java:190)
at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:93)
at org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:688)
at org.apache.lucene.index.IndexWriter$ReaderPool.writeSomeDocValuesUpdates(IndexWriter.java:703)
at org.apache.lucene.index.FrozenBufferedUpdates.apply(FrozenBufferedUpdates.java:331)
at org.apache.lucene.index.DocumentsWriter$ResolveUpdatesEvent.process(DocumentsWriter.java:723)
at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5060)
at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5048)
at org.apache.lucene.index.IndexWriter.updateDocValues(IndexWriter.java:1879)
at org.apache.lucene.index.TestBinaryDocValuesUpdates.testTonsOfUpdates(TestBinaryDocValuesUpdates.java:1323)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI
Error Message:
Something is broken in the assert for no shards using the same indexDir - probably something was changed in the attributes published in the MBean of SolrCore : {}
Stack Trace:
java.lang.AssertionError: Something is broken in the assert for no shards using the same indexDir - probably something was changed in the attributes published in the MBean of SolrCore : {}
at __randomizedtesting.SeedInfo.seed([B2503D85157AD516:FA2549311349FA83]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.checkNoTwoShardsUseTheSameIndexDir(CollectionsAPIDistributedZkTest.java:580)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:461)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.HttpPartitionTest.test
Error Message:
Didn't see all replicas for shard shard1 in c8n_1x2 come up within 90000 ms! ClusterState: { "collMinRf_1x3":{ "pullReplicas":"0", "replicationFactor":"1", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{ "core_node1":{ "core":"collMinRf_1x3_shard1_replica_t1", "base_url":"http://127.0.0.1:47250/mczk/g", "node_name":"127.0.0.1:47250_mczk%2Fg", "state":"active", "type":"TLOG"}, "core_node2":{ "core":"collMinRf_1x3_shard1_replica_t2", "base_url":"http://127.0.0.1:39241/mczk/g", "node_name":"127.0.0.1:39241_mczk%2Fg", "state":"active", "type":"TLOG", "leader":"true"}, "core_node3":{ "core":"collMinRf_1x3_shard1_replica_t3", "base_url":"http://127.0.0.1:34423/mczk/g", "node_name":"127.0.0.1:34423_mczk%2Fg", "state":"active", "type":"TLOG"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"0", "tlogReplicas":"3"}, "collection1":{ "pullReplicas":"0", "replicationFactor":"1", "shards":{ "shard1":{ "range":"80000000-ffffffff", "state":"active", "replicas":{"core_node2":{ "core":"collection1_shard1_replica_t0", "base_url":"http://127.0.0.1:39241/mczk/g", "node_name":"127.0.0.1:39241_mczk%2Fg", "state":"active", "type":"TLOG", "leader":"true"}}}, "shard2":{ "range":"0-7fffffff", "state":"active", "replicas":{ "core_node1":{ "core":"collection1_shard2_replica_t0", "base_url":"http://127.0.0.1:36830/mczk/g", "node_name":"127.0.0.1:36830_mczk%2Fg", "state":"active", "type":"TLOG", "leader":"true"}, "core_node3":{ "core":"collection1_shard2_replica_t1", "base_url":"http://127.0.0.1:34423/mczk/g", "node_name":"127.0.0.1:34423_mczk%2Fg", "state":"active", "type":"TLOG"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"1", "tlogReplicas":"0"}, "control_collection":{ "pullReplicas":"0", "replicationFactor":"1", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{"core_node1":{ "core":"control_collection_shard1_replica_n1", "base_url":"http://127.0.0.1:47250/mczk/g", "node_name":"127.0.0.1:47250_mczk%2Fg", "state":"active", "type":"NRT", "leader":"true"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"1", "tlogReplicas":"0"}, "c8n_1x2":{ "pullReplicas":"0", "replicationFactor":"1", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{ "core_node1":{ "core":"c8n_1x2_shard1_replica_t1", "base_url":"http://127.0.0.1:36830/mczk/g", "node_name":"127.0.0.1:36830_mczk%2Fg", "state":"active", "type":"TLOG", "leader":"true"}, "core_node2":{ "state":"down", "base_url":"http://127.0.0.1:39241/mczk/g", "core":"c8n_1x2_shard1_replica_t2", "node_name":"127.0.0.1:39241_mczk%2Fg", "type":"TLOG"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"0", "tlogReplicas":"2"}}
Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x2 come up within 90000 ms! ClusterState: {
"collMinRf_1x3":{
"pullReplicas":"0",
"replicationFactor":"1",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{
"core_node1":{
"core":"collMinRf_1x3_shard1_replica_t1",
"base_url":"http://127.0.0.1:47250/mczk/g",
"node_name":"127.0.0.1:47250_mczk%2Fg",
"state":"active",
"type":"TLOG"},
"core_node2":{
"core":"collMinRf_1x3_shard1_replica_t2",
"base_url":"http://127.0.0.1:39241/mczk/g",
"node_name":"127.0.0.1:39241_mczk%2Fg",
"state":"active",
"type":"TLOG",
"leader":"true"},
"core_node3":{
"core":"collMinRf_1x3_shard1_replica_t3",
"base_url":"http://127.0.0.1:34423/mczk/g",
"node_name":"127.0.0.1:34423_mczk%2Fg",
"state":"active",
"type":"TLOG"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"0",
"tlogReplicas":"3"},
"collection1":{
"pullReplicas":"0",
"replicationFactor":"1",
"shards":{
"shard1":{
"range":"80000000-ffffffff",
"state":"active",
"replicas":{"core_node2":{
"core":"collection1_shard1_replica_t0",
"base_url":"http://127.0.0.1:39241/mczk/g",
"node_name":"127.0.0.1:39241_mczk%2Fg",
"state":"active",
"type":"TLOG",
"leader":"true"}}},
"shard2":{
"range":"0-7fffffff",
"state":"active",
"replicas":{
"core_node1":{
"core":"collection1_shard2_replica_t0",
"base_url":"http://127.0.0.1:36830/mczk/g",
"node_name":"127.0.0.1:36830_mczk%2Fg",
"state":"active",
"type":"TLOG",
"leader":"true"},
"core_node3":{
"core":"collection1_shard2_replica_t1",
"base_url":"http://127.0.0.1:34423/mczk/g",
"node_name":"127.0.0.1:34423_mczk%2Fg",
"state":"active",
"type":"TLOG"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"1",
"tlogReplicas":"0"},
"control_collection":{
"pullReplicas":"0",
"replicationFactor":"1",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{"core_node1":{
"core":"control_collection_shard1_replica_n1",
"base_url":"http://127.0.0.1:47250/mczk/g",
"node_name":"127.0.0.1:47250_mczk%2Fg",
"state":"active",
"type":"NRT",
"leader":"true"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"1",
"tlogReplicas":"0"},
"c8n_1x2":{
"pullReplicas":"0",
"replicationFactor":"1",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{
"core_node1":{
"core":"c8n_1x2_shard1_replica_t1",
"base_url":"http://127.0.0.1:36830/mczk/g",
"node_name":"127.0.0.1:36830_mczk%2Fg",
"state":"active",
"type":"TLOG",
"leader":"true"},
"core_node2":{
"state":"down",
"base_url":"http://127.0.0.1:39241/mczk/g",
"core":"c8n_1x2_shard1_replica_t2",
"node_name":"127.0.0.1:39241_mczk%2Fg",
"type":"TLOG"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"0",
"tlogReplicas":"2"}}
at __randomizedtesting.SeedInfo.seed([B2503D85157AD516:3A04025FBB86B8EE]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1996)
at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:368)
at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.testCollectionsAPI
Error Message:
Something is broken in the assert for no shards using the same indexDir - probably something was changed in the attributes published in the MBean of SolrCore : {}
Stack Trace:
java.lang.AssertionError: Something is broken in the assert for no shards using the same indexDir - probably something was changed in the attributes published in the MBean of SolrCore : {}
at __randomizedtesting.SeedInfo.seed([B2503D85157AD516:FA2549311349FA83]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.checkNoTwoShardsUseTheSameIndexDir(CollectionsAPIDistributedZkTest.java:580)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:461)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.core.TestJmxIntegration
Error Message:
JMX not enabled
Stack Trace:
java.lang.AssertionError: JMX not enabled
at __randomizedtesting.SeedInfo.seed([B2503D85157AD516]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.core.TestJmxIntegration.beforeClass(TestJmxIntegration.java:75)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:847)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
Timeout waiting for all live and active
Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
at __randomizedtesting.SeedInfo.seed([B2503D85157AD516:3A04025FBB86B8EE]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:317)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 1396 lines...]
[junit4] Suite: org.apache.lucene.index.TestBinaryDocValuesUpdates
[junit4] 1> TEST: isNRT=true reader1=StandardDirectoryReader(segments:3:nrt _0(7.0.0):c2)
[junit4] 1> TEST: now reopen
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestBinaryDocValuesUpdates -Dtests.method=testTonsOfUpdates -Dtests.seed=A3EFB8732F26DD8 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=cs -Dtests.timezone=Atlantic/South_Georgia -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 4.26s J0 | TestBinaryDocValuesUpdates.testTonsOfUpdates <<<
[junit4] > Throwable #1: java.io.FileNotFoundException: _2_4.fnm in dir=RAMDirectory@614bdb2f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@448315ab
[junit4] > at __randomizedtesting.SeedInfo.seed([A3EFB8732F26DD8:721B258CD0D2423A]:0)
[junit4] > at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:750)
[junit4] > at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:119)
[junit4] > at org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1072)
[junit4] > at org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat.read(Lucene60FieldInfosFormat.java:113)
[junit4] > at org.apache.lucene.index.SegmentReader.initFieldInfos(SegmentReader.java:190)
[junit4] > at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:93)
[junit4] > at org.apache.lucene.index.ReadersAndUpdates.writeFieldUpdates(ReadersAndUpdates.java:688)
[junit4] > at org.apache.lucene.index.IndexWriter$ReaderPool.writeSomeDocValuesUpdates(IndexWriter.java:703)
[junit4] > at org.apache.lucene.index.FrozenBufferedUpdates.apply(FrozenBufferedUpdates.java:331)
[junit4] > at org.apache.lucene.index.DocumentsWriter$ResolveUpdatesEvent.process(DocumentsWriter.java:723)
[junit4] > at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5060)
[junit4] > at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5048)
[junit4] > at org.apache.lucene.index.IndexWriter.updateDocValues(IndexWriter.java:1879)
[junit4] > at org.apache.lucene.index.TestBinaryDocValuesUpdates.testTonsOfUpdates(TestBinaryDocValuesUpdates.java:1323)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/test/J0/temp/lucene.index.TestBinaryDocValuesUpdates_A3EFB8732F26DD8-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {bdv=PostingsFormat(name=LuceneVarGapDocFreqInterval), f=Lucene50(blocksize=128), k1=Lucene50(blocksize=128), dvUpdateKey=PostingsFormat(name=LuceneVarGapDocFreqInterval), foo=Lucene50(blocksize=128), k2=PostingsFormat(name=Direct), upd=PostingsFormat(name=Memory), updKey=Lucene50(blocksize=128), id=PostingsFormat(name=Direct), key=PostingsFormat(name=Direct)}, docValues:{ndv=DocValuesFormat(name=Memory), f10=DocValuesFormat(name=Lucene70), f12=DocValuesFormat(name=Lucene70), f11=DocValuesFormat(name=Asserting), f14=DocValuesFormat(name=Lucene70), f13=DocValuesFormat(name=Memory), f0=DocValuesFormat(name=Asserting), f16=DocValuesFormat(name=Lucene70), f1=DocValuesFormat(name=Lucene70), f15=DocValuesFormat(name=Asserting), f2=DocValuesFormat(name=Memory), f18=DocValuesFormat(name=Lucene70), f17=DocValuesFormat(name=Memory), f3=DocValuesFormat(name=Lucene70), f4=DocValuesFormat(name=Asserting), f19=DocValuesFormat(name=Asserting), f5=DocValuesFormat(name=Lucene70), bdv2=DocValuesFormat(name=Asserting), f6=DocValuesFormat(name=Memory), number=DocValuesFormat(name=Lucene70), f7=DocValuesFormat(name=Lucene70), f8=DocValuesFormat(name=Asserting), bdv1=DocValuesFormat(name=Lucene70), f9=DocValuesFormat(name=Lucene70), id=DocValuesFormat(name=Lucene70), val=DocValuesFormat(name=Lucene70), f21=DocValuesFormat(name=Lucene70), f20=DocValuesFormat(name=Memory), f23=DocValuesFormat(name=Lucene70), f22=DocValuesFormat(name=Asserting), f25=DocValuesFormat(name=Lucene70), upd=DocValuesFormat(name=Lucene70), f24=DocValuesFormat(name=Memory), sort=DocValuesFormat(name=Asserting), f26=DocValuesFormat(name=Asserting), cf0=DocValuesFormat(name=Lucene70), cf2=DocValuesFormat(name=Lucene70), cf1=DocValuesFormat(name=Asserting), cf4=DocValuesFormat(name=Lucene70), cf3=DocValuesFormat(name=Memory), cf6=DocValuesFormat(name=Lucene70), cf5=DocValuesFormat(name=Asserting), cf8=DocValuesFormat(name=Lucene70), cf7=DocValuesFormat(name=Memory), cf9=DocValuesFormat(name=Asserting), ssdv=DocValuesFormat(name=Asserting), sdv=DocValuesFormat(name=Lucene70), cf25=DocValuesFormat(name=Asserting), cf26=DocValuesFormat(name=Lucene70), cf23=DocValuesFormat(name=Memory), cf24=DocValuesFormat(name=Lucene70), cf21=DocValuesFormat(name=Asserting), cf22=DocValuesFormat(name=Lucene70), cf20=DocValuesFormat(name=Lucene70), key=DocValuesFormat(name=Lucene70), cf=DocValuesFormat(name=Lucene70), cf18=DocValuesFormat(name=Asserting), cf19=DocValuesFormat(name=Lucene70), f=DocValuesFormat(name=Asserting), cf16=DocValuesFormat(name=Memory), cf17=DocValuesFormat(name=Lucene70), cf14=DocValuesFormat(name=Asserting), cf15=DocValuesFormat(name=Lucene70), control=DocValuesFormat(name=Lucene70), cf12=DocValuesFormat(name=Memory), cf13=DocValuesFormat(name=Lucene70), cf10=DocValuesFormat(name=Asserting), cf11=DocValuesFormat(name=Lucene70), bdv=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1358, maxMBSortInHeap=6.525495371582023, sim=RandomSimilarity(queryNorm=false): {}, locale=cs, timezone=Atlantic/South_Georgia
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=341529880,total=450363392
[junit4] 2> NOTE: All tests run in this JVM: [TestBooleanQueryVisitSubscorers, TestCharFilter, TestFieldMaskingSpanQuery, TestCodecHoldsOpenFiles, TestDirectoryReaderReopen, TestBinaryTerms, TestDocsWithFieldSet, TestDateTools, TestIsCurrent, TestReqExclBulkScorer, TestStringMSBRadixSorter, TestMatchAllDocsQuery, TestRoaringDocIdSet, TestIndexReaderClose, TestDeletionPolicy, TestMaxTermFrequency, TestSortedNumericSortField, TestIndexableField, TestDocIdSetBuilder, TestCharArrayMap, TestUpgradeIndexMergePolicy, TestNamedSPILoader, Test4GBStoredFields, TestSpanMultiTermQueryWrapper, TestStressDeletes, TestDocsAndPositions, TestUniqueTermCount, TestOmitNorms, TestLucene70SegmentInfoFormat, TestPerSegmentDeletes, TestBytesStore, TestSort, TestIndexWriterOnVMError, TestMergeRateLimiter, TestBooleanRewrites, TestApproximationSearchEquivalence, TestSearchAfter, TestTotalHitCountCollector, TestTrackingDirectoryWrapper, TestGeoUtils, TestOrdinalMap, TestSpansEnum, TestBooleanMinShouldMatch, TestSmallFloat, TestNewestSegment, TestClassicSimilarity, Test2BPoints, TestDelegatingAnalyzerWrapper, TestMultiTermQueryRewrites, TestMultiPhraseQuery, TestNGramPhraseQuery, TestIndexWriterNRTIsCurrent, TestFrequencyTrackingRingBuffer, TestTerm, TestManyFields, TestPostingsOffsets, TestNeedsScores, TestExitableDirectoryReader, TestSearch, TestBooleanScorer, Test2BTerms, TestAllFilesCheckIndexHeader, TestDocValuesIndexing, TestTopFieldCollector, TestLucene50TermVectorsFormat, TestIndexWriterExceptions, TestUTF32ToUTF8, TestFlex, TestWildcardRandom, TestOfflineSorter, TestFloatRangeFieldQueries, TestIndexingSequenceNumbers, TestTerms, TestSparseFixedBitDocIdSet, TestAutomatonQuery, TestNeverDelete, TestLongPostings, TestStressIndexing, TestSnapshotDeletionPolicy, TestIndexWriterUnicode, TestIndexWriterForceMerge, TestPerFieldPostingsFormat, TestArrayUtil, TestLevenshteinAutomata, TestByteSlices, TestTermsEnum, TestFixedBitSet, TestComplexExplanations, TestRegexpRandom2, TestNRTCachingDirectory, TestFieldCacheRewriteMethod, TestPagedBytes, TestTransactions, TestCrash, TestMultiThreadTermVectors, TestPerFieldPostingsFormat2, TestSimpleSearchEquivalence, TestSimilarityBase, TestDeterminizeLexicon, TestSpans, TestMixedCodecs, TestStressIndexing2, TestDeterminism, TestTermVectorsWriter, TestCodecs, TestParallelLeafReader, TestBytesRefHash, TestThreadedForceMerge, TestStressNRT, TestIndexInput, TestForceMergeForever, TestOmitTf, TestNearSpansOrdered, TestMultiFields, TestTermRangeQuery, TestDocumentsWriterDeleteQueue, TestDocumentWriter, TestCrashCausesCorruptIndex, TestScorerPerf, TestForTooMuchCloning, TestSpanSearchEquivalence, TestAttributeSource, TestNoMergeScheduler, TestDemo, TestCloseableThreadLocal, TestScoreCachingWrappingScorer, TestSimpleAttributeImpl, Test2BPostingsBytes, TestIOUtils, TestCharArraySet, TestStopFilter, TestCodecUtil, TestBlockPostingsFormat2, TestBlockPostingsFormat3, TestForUtil, TestLucene50CompoundFormat, TestLucene50StoredFieldsFormatHighCompression, TestAllFilesHaveChecksumFooter, TestBinaryDocValuesUpdates]
[junit4] Completed [349/454 (1!)] on J0 in 25.05s, 29 tests, 1 error <<< FAILURES!
[...truncated 10839 lines...]
[junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_B2503D85157AD516-001/init-core-data-001
[junit4] 2> 2199033 WARN (SUITE-CollectionsAPIDistributedZkTest-seed#[B2503D85157AD516]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=36 numCloses=36
[junit4] 2> 2199033 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[B2503D85157AD516]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 2199035 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[B2503D85157AD516]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 2199036 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[B2503D85157AD516]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_B2503D85157AD516-001/tempDir-001
[junit4] 2> 2199036 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[B2503D85157AD516]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2199037 INFO (Thread-7221) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2199037 INFO (Thread-7221) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2199039 ERROR (Thread-7221) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2199137 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[B2503D85157AD516]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:38620
[junit4] 2> 2199149 INFO (jetty-launcher-1417-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2199149 INFO (jetty-launcher-1417-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2199151 INFO (jetty-launcher-1417-thread-3) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2199153 INFO (jetty-launcher-1417-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7998ace6{/solr,null,AVAILABLE}
[junit4] 2> 2199154 INFO (jetty-launcher-1417-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6b9a7ef3{SSL,[ssl, http/1.1]}{127.0.0.1:43086}
[junit4] 2> 2199154 INFO (jetty-launcher-1417-thread-2) [ ] o.e.j.s.Server Started @2202092ms
[junit4] 2> 2199154 INFO (jetty-launcher-1417-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43086}
[junit4] 2> 2199154 ERROR (jetty-launcher-1417-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2199154 INFO (jetty-launcher-1417-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2199154 INFO (jetty-launcher-1417-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2199154 INFO (jetty-launcher-1417-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2199155 INFO (jetty-launcher-1417-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-15T12:25:09.528Z
[junit4] 2> 2199163 INFO (jetty-launcher-1417-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1cef6622{/solr,null,AVAILABLE}
[junit4] 2> 2199164 INFO (jetty-launcher-1417-thread-4) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2199164 INFO (jetty-launcher-1417-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@12d07685{SSL,[ssl, http/1.1]}{127.0.0.1:57660}
[junit4] 2> 2199164 INFO (jetty-launcher-1417-thread-3) [ ] o.e.j.s.Server Started @2202101ms
[junit4] 2> 2199164 INFO (jetty-launcher-1417-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57660}
[junit4] 2> 2199164 ERROR (jetty-launcher-1417-thread-3) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2199164 INFO (jetty-launcher-1417-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2199164 INFO (jetty-launcher-1417-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2199164 INFO (jetty-launcher-1417-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2199164 INFO (jetty-launcher-1417-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-15T12:25:09.538Z
[junit4] 2> 2199166 INFO (jetty-launcher-1417-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4cfbf0dd{/solr,null,AVAILABLE}
[junit4] 2> 2199167 INFO (jetty-launcher-1417-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2baba1e7{SSL,[ssl, http/1.1]}{127.0.0.1:53052}
[junit4] 2> 2199167 INFO (jetty-launcher-1417-thread-1) [ ] o.e.j.s.Server Started @2202104ms
[junit4] 2> 2199167 INFO (jetty-launcher-1417-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=53052}
[junit4] 2> 2199167 ERROR (jetty-launcher-1417-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2199167 INFO (jetty-launcher-1417-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2199167 INFO (jetty-launcher-1417-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2199167 INFO (jetty-launcher-1417-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2199167 INFO (jetty-launcher-1417-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-15T12:25:09.541Z
[junit4] 2> 2199171 INFO (jetty-launcher-1417-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4fe8b9ef{/solr,null,AVAILABLE}
[junit4] 2> 2199171 INFO (jetty-launcher-1417-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4ca0dfae{SSL,[ssl, http/1.1]}{127.0.0.1:37012}
[junit4] 2> 2199171 INFO (jetty-launcher-1417-thread-4) [ ] o.e.j.s.Server Started @2202109ms
[junit4] 2> 2199171 INFO (jetty-launcher-1417-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37012}
[junit4] 2> 2199171 ERROR (jetty-launcher-1417-thread-4) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2199172 INFO (jetty-launcher-1417-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2199172 INFO (jetty-launcher-1417-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2199172 INFO (jetty-launcher-1417-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2199172 INFO (jetty-launcher-1417-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-15T12:25:09.546Z
[junit4] 2> 2199173 INFO (jetty-launcher-1417-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2199186 INFO (jetty-launcher-1417-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2199187 INFO (jetty-launcher-1417-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38620/solr
[junit4] 2> 2199195 INFO (jetty-launcher-1417-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2199195 INFO (jetty-launcher-1417-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2199195 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15de5dad2230005, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2199200 INFO (jetty-launcher-1417-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38620/solr
[junit4] 2> 2199226 INFO (jetty-launcher-1417-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38620/solr
[junit4] 2> 2199228 INFO (jetty-launcher-1417-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38620/solr
[junit4] 2> 2199240 INFO (jetty-launcher-1417-thread-1) [n:127.0.0.1:53052_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2199240 INFO (jetty-launcher-1417-thread-2) [n:127.0.0.1:43086_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2199240 INFO (jetty-launcher-1417-thread-1) [n:127.0.0.1:53052_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:53052_solr
[junit4] 2> 2199241 INFO (jetty-launcher-1417-thread-1) [n:127.0.0.1:53052_solr ] o.a.s.c.Overseer Overseer (id=98487494863159306-127.0.0.1:53052_solr-n_0000000000) starting
[junit4] 2> 2199241 INFO (jetty-launcher-1417-thread-2) [n:127.0.0.1:43086_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43086_solr
[junit4] 2> 2199250 INFO (zkCallback-1438-thread-1-processing-n:127.0.0.1:53052_solr) [n:127.0.0.1:53052_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2199250 INFO (zkCallback-1439-thread-1-processing-n:127.0.0.1:43086_solr) [n:127.0.0.1:43086_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2199257 INFO (jetty-launcher-1417-thread-1) [n:127.0.0.1:53052_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53052_solr
[junit4] 2> 2199258 INFO (zkCallback-1439-thread-1-processing-n:127.0.0.1:43086_solr) [n:127.0.0.1:43086_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2199258 INFO (zkCallback-1438-thread-1-processing-n:127.0.0.1:53052_solr) [n:127.0.0.1:53052_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2199258 INFO (jetty-launcher-1417-thread-4) [n:127.0.0.1:37012_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2199266 INFO (jetty-launcher-1417-thread-4) [n:127.0.0.1:37012_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2199267 INFO (jetty-launcher-1417-thread-4) [n:127.0.0.1:37012_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37012_solr
[junit4] 2> 2199268 INFO (zkCallback-1439-thread-1-processing-n:127.0.0.1:43086_solr) [n:127.0.0.1:43086_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2199269 INFO (zkCallback-1438-thread-1-processing-n:127.0.0.1:53052_solr) [n:127.0.0.1:53052_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2199274 INFO (jetty-launcher-1417-thread-3) [n:127.0.0.1:57660_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2199274 INFO (jetty-launcher-1417-thread-3) [n:127.0.0.1:57660_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2199275 INFO (jetty-launcher-1417-thread-3) [n:127.0.0.1:57660_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57660_solr
[junit4] 2> 2199276 INFO (zkCallback-1439-thread-1-processing-n:127.0.0.1:43086_solr) [n:127.0.0.1:43086_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2199285 INFO (zkCallback-1438-thread-2-processing-n:127.0.0.1:53052_solr) [n:127.0.0.1:53052_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2199290 INFO (zkCallback-1427-thread-1-processing-n:127.0.0.1:57660_solr) [n:127.0.0.1:57660_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2199291 INFO (zkCallback-1440-thread-1-processing-n:127.0.0.1:37012_solr) [n:127.0.0.1:37012_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
[junit4] 2> 2199355 WARN (jetty-launcher-1417-thread-2) [n:127.0.0.1:43086_solr ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
[junit4] 2> 2199356 WARN (jetty-launcher-1417-thread-2) [n:127.0.0.1:43086_solr ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
[junit4] 2> 2199356 WARN (jetty-launcher-1417-thread-2) [n:127.0.0.1:43086_solr ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
[junit4] 2> 2199356 INFO (jetty-launcher-1417-thread-2) [n:127.0.0.1:43086_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_B2503D85157AD516-001/tempDir-001/node2
[junit4] 2> 2199363 WARN (jetty-launcher-1417-thread-4) [n:127.0.0.1:37012_solr ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
[junit4] 2> 2199364 WARN (jetty-launcher-1417-thread-4) [n:127.0.0.1:37012_solr ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
[junit4] 2> 2199364 WARN (jetty-launcher-1417-thread-4) [n:127.0.0.1:37012_solr ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
[junit4] 2> 2199364 INFO (jetty-launcher-1417-thread-4) [n:127.0.0.1:37012_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_B2503D85157AD516-001/tempDir-001/node4
[junit4] 2> 2199374 WARN (jetty-launcher-1417-thread-1) [n:127.0.0.1:53052_solr ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
[junit4] 2> 2199374 WARN (jetty-launcher-1417-thread-1) [n:127.0.0.1:53052_solr ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
[junit4] 2> 2199374 WARN (jetty-launcher-1417-thread-1) [n:127.0.0.1:53052_solr ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
[junit4] 2> 2199375 INFO (jetty-launcher-1417-thread-1) [n:127.0.0.1:53052_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_B2503D85157AD516-001/tempDir-001/node1
[junit4] 2> 2199406 WARN (jetty-launcher-1417-thread-3) [n:127.0.0.1:57660_solr ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
[junit4] 2> 2199406 WARN (jetty-launcher-1417-thread-3) [n:127.0.0.1:57660_solr ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
[junit4] 2> 2199407 WARN (jetty-launcher-1417-thread-3) [n:127.0.0.1:57660_solr ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
[junit4] 2> 2199407 INFO (jetty-launcher-1417-thread-3) [n:127.0.0.1:57660_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_B2503D85157AD516-001/tempDir-001/node3
[junit4] 2> 2199417 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[B2503D85157AD516]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 2199418 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[B2503D85157AD516]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38620/solr ready
[junit4] 2> 2199476 INFO (TEST-CollectionsAPIDistributedZkTest.deletePartiallyCreatedCollection-seed#[B2503D85157AD516]) [ ] o.a.s.SolrTestCaseJ4 ###Starting deletePartiallyCreatedCollection
[junit4] 2> 2199476 INFO (TEST-CollectionsAPIDistributedZkTest.deletePartiallyCreatedCollection-seed#[B2503D85157AD516]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 2199500 INFO (qtp566194340-17522) [n:127.0.0.1:37012_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=halfdeletedcollection&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2199505 INFO (OverseerThreadFactory-3943-thread-1-processing-n:127.0.0.1:53052_solr) [n:127.0.0.1:53052_solr ] o.a.s.c.CreateCollectionCmd Create collection halfdeletedcollection
[junit4] 2> 2199505 WARN (OverseerThreadFactory-3943-thread-1-processing-n:127.0.0.1:53052_solr) [n:127.0.0.1:53052_solr ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (halfdeletedcollection) without cores.
[junit4] 2> 2199613 INFO (qtp566194340-17522) [n:127.0.0.1:37012_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2199613 INFO (qtp566194340-17522) [n:127.0.0.1:37012_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=halfdeletedcollection&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&wt=javabin&version=2} status=0 QTime=113
[junit4] 2> 2199615 INFO (qtp566194340-17523) [n:127.0.0.1:37012_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params dataDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_B2503D85157AD516-001/tempDir-002&action=ADDREPLICA&collection=halfdeletedcollection&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2199616 INFO (OverseerCollectionConfigSetProcessor-98487494863159306-127.0.0.1:53052_solr-n_0000000000) [n:127.0.0.1:53052_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2199617 INFO (OverseerThreadFactory-3943-thread-2-processing-n:127.0.0.1:53052_solr) [n:127.0.0.1:53052_solr ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:37012_solr for creating new replica
[junit4] 2> 2199618 INFO (OverseerStateUpdate-98487494863159306-127.0.0.1:53052_solr-n_0000000000) [n:127.0.0.1:53052_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"addreplica",
[junit4] 2> "collection":"halfdeletedcollection",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"halfdeletedcollection_shard1_replica_n0",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:37012/solr",
[junit4] 2> "node_name":"127.0.0.1:37012_solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 2199824 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr ] o.a.s.u.TestInjection Inject random core creation delay of 8s
[junit4] 2> 2207824 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node1&dataDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_B2503D85157AD516-001/tempDir-002&collection.configName=conf&name=halfdeletedcollection_shard1_replica_n0&action=CREATE&collection=halfdeletedcollection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2207825 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2207837 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2207843 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.s.IndexSchema [halfdeletedcollection_shard1_replica_n0] Schema name=minimal
[junit4] 2> 2207845 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2207845 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.c.CoreContainer Creating SolrCore 'halfdeletedcollection_shard1_replica_n0' using configuration from collection halfdeletedcollection, trusted=true
[junit4] 2> 2207846 WARN (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
[junit4] 2> 2207846 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2207847 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.c.SolrCore [[halfdeletedcollection_shard1_replica_n0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_B2503D85157AD516-001/tempDir-001/node4/halfdeletedcollection_shard1_replica_n0], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_B2503D85157AD516-001/tempDir-002/]
[junit4] 2> 2207881 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2207881 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2207882 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2207882 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2207883 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.s.SolrIndexSearcher Opening [Searcher@9bcaef9[halfdeletedcollection_shard1_replica_n0] main]
[junit4] 2> 2207884 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2207884 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2207884 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2207885 INFO (searcherExecutor-3952-thread-1-processing-n:127.0.0.1:37012_solr x:halfdeletedcollection_shard1_replica_n0 s:shard1 c:halfdeletedcollection r:core_node1) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.c.SolrCore [halfdeletedcollection_shard1_replica_n0] Registered new searcher Searcher@9bcaef9[halfdeletedcollection_shard1_replica_n0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2207885 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1575799927088349184
[junit4] 2> 2207892 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2207892 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2207892 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:37012/solr/halfdeletedcollection_shard1_replica_n0/
[junit4] 2> 2207892 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2207892 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.c.SyncStrategy https://127.0.0.1:37012/solr/halfdeletedcollection_shard1_replica_n0/ has no replicas
[junit4] 2> 2207892 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2207896 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:37012/solr/halfdeletedcollection_shard1_replica_n0/ shard1
[junit4] 2> 2207997 INFO (zkCallback-1440-thread-1-processing-n:127.0.0.1:37012_solr) [n:127.0.0.1:37012_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfdeletedcollection/state.json] for collection [halfdeletedcollection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2208047 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2208050 INFO (qtp566194340-17520) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node1&dataDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_B2503D85157AD516-001/tempDir-002&collection.configName=conf&name=halfdeletedcollection_shard1_replica_n0&action=CREATE&collection=halfdeletedcollection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=8226
[junit4] 2> 2208052 INFO (qtp566194340-17523) [n:127.0.0.1:37012_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={dataDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_B2503D85157AD516-001/tempDir-002&action=ADDREPLICA&collection=halfdeletedcollection&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=8437
[junit4] 2> 2208053 INFO (qtp566194340-17539) [n:127.0.0.1:37012_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=halfdeletedcollection&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2208058 INFO (OverseerCollectionConfigSetProcessor-98487494863159306-127.0.0.1:53052_solr-n_0000000000) [n:127.0.0.1:53052_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2208058 INFO (OverseerThreadFactory-3943-thread-3-processing-n:127.0.0.1:53052_solr) [n:127.0.0.1:53052_solr ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
[junit4] 2> 2208059 INFO (qtp566194340-17521) [n:127.0.0.1:37012_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.halfdeletedcollection.shard1.replica_n0, tag=null
[junit4] 2> 2208059 INFO (qtp566194340-17521) [n:127.0.0.1:37012_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7070e062: rootName = null, domain = solr.core.halfdeletedcollection.shard1.replica_n0, service url = null, agent id = null] for registry solr.core.halfdeletedcollection.shard1.replica_n0 / null
[junit4] 2> 2208061 INFO (qtp566194340-17521) [n:127.0.0.1:37012_solr ] o.a.s.c.SolrCore [halfdeletedcollection_shard1_replica_n0] CLOSING SolrCore org.apache.solr.core.SolrCore@81f07d3
[junit4] 2> 2208064 INFO (qtp566194340-17521) [n:127.0.0.1:37012_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.halfdeletedcollection.shard1.replica_n0, tag=136251347
[junit4] 2> 2208064 INFO (qtp566194340-17521) [n:127.0.0.1:37012_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.halfdeletedcollection.shard1.leader, tag=136251347
[junit4] 2> 2208066 INFO (qtp566194340-17521) [n:127.0.0.1:37012_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=halfdeletedcollection_shard1_replica_n0&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=7
[junit4] 2> 2208771 INFO (qtp566194340-17539) [n:127.0.0.1:37012_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=halfdeletedcollection&action=DELETE&wt=javabin&version=2} status=0 QTime=717
[junit4] 2> 2208773 INFO (qtp566194340-17522) [n:127.0.0.1:37012_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2208773 INFO (qtp566194340-17522) [n:127.0.0.1:37012_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2208774 INFO (qtp566194340-17522) [n:127.0.0.1:37012_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=halfdeletedcollection&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2208776 INFO (OverseerCollectionConfigSetProcessor-98487494863159306-127.0.0.1:53052_solr-n_0000000000) [n:127.0.0.1:53052_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2208776 INFO (OverseerThreadFactory-3943-thread-4-processing-n:127.0.0.1:53052_solr) [n:127.0.0.1:53052_solr ] o.a.s.c.CreateCollectionCmd Create collection halfdeletedcollection
[junit4] 2> 2208982 INFO (OverseerStateUpdate-98487494863159306-127.0.0.1:53052_solr-n_0000000000) [n:127.0.0.1:53052_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"halfdeletedcollection",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"halfdeletedcollection_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:43086/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 2208983 INFO (OverseerStateUpdate-98487494863159306-127.0.0.1:53052_solr-n_0000000000) [n:127.0.0.1:53052_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"halfdeletedcollection",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"halfdeletedcollection_shard2_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:37012/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 2209192 INFO (qtp566194340-17518) [n:127.0.0.1:37012_solr ] o.a.s.u.TestInjection Inject random core creation delay of 8s
[junit4] 2> 2209213 INFO (qtp1576156728-17501) [n:127.0.0.1:43086_solr ] o.a.s.u.TestInjection Inject random core creation delay of 8s
[junit4] 2> 2217192 INFO (qtp566194340-17518) [n:127.0.0.1:37012_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=halfdeletedcollection_shard2_replica_n1&action=CREATE&numShards=2&collection=halfdeletedcollection&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2217198 INFO (zkCallback-1440-thread-1-processing-n:127.0.0.1:37012_solr) [n:127.0.0.1:37012_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfdeletedcollection/state.json] for collection [halfdeletedcollection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2217198 INFO (zkCallback-1440-thread-2-processing-n:127.0.0.1:37012_solr) [n:127.0.0.1:37012_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfdeletedcollection/state.json] for collection [halfdeletedcollection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2217213 INFO (qtp1576156728-17501) [n:127.0.0.1:43086_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node1&collection.configName=conf&newCollection=true&name=halfdeletedcollection_shard1_replica_n1&action=CREATE&numShards=2&collection=halfdeletedcollection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2217214 INFO (qtp1576156728-17501) [n:127.0.0.1:43086_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2217319 INFO (zkCallback-1440-thread-2-processing-n:127.0.0.1:37012_solr) [n:127.0.0.1:37012_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfdeletedcollection/state.json] for collection [halfdeletedcollection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2217319 INFO (zkCallback-1440-thread-1-processing-n:127.0.0.1:37012_solr) [n:127.0.0.1:37012_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfdeletedcollection/state.json] for collection [halfdeletedcollection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2217319 INFO (zkCallback-1439-thread-1-processing-n:127.0.0.1:43086_solr) [n:127.0.0.1:43086_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfdeletedcollection/state.json] for collection [halfdeletedcollection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2217319 INFO (zkCallback-1439-thread-2-processing-n:127.0.0.1:43086_solr) [n:127.0.0.1:43086_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfdeletedcollection/state.json] for collection [halfdeletedcollection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2218208 INFO (qtp566194340-17518) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard2 r:core_node2 x:halfdeletedcollection_shard2_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2218214 INFO (qtp566194340-17518) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard2 r:core_node2 x:halfdeletedcollection_shard2_replica_n1] o.a.s.s.IndexSchema [halfdeletedcollection_shard2_replica_n1] Schema name=minimal
[junit4] 2> 2218217 INFO (qtp566194340-17518) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard2 r:core_node2 x:halfdeletedcollection_shard2_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2218217 INFO (qtp566194340-17518) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard2 r:core_node2 x:halfdeletedcollection_shard2_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'halfdeletedcollection_shard2_replica_n1' using configuration from collection halfdeletedcollection, trusted=true
[junit4] 2> 2218217 WARN (qtp566194340-17518) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard2 r:core_node2 x:halfdeletedcollection_shard2_replica_n1] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
[junit4] 2> 2218217 INFO (qtp566194340-17518) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard2 r:core_node2 x:halfdeletedcollection_shard2_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2218217 INFO (qtp566194340-17518) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard2 r:core_node2 x:halfdeletedcollection_shard2_replica_n1] o.a.s.c.SolrCore [[halfdeletedcollection_shard2_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_B2503D85157AD516-001/tempDir-001/node4/halfdeletedcollection_shard2_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_B2503D85157AD516-001/tempDir-001/node4/halfdeletedcollection_shard2_replica_n1/data/]
[junit4] 2> 2218226 INFO (qtp1576156728-17501) [n:127.0.0.1:43086_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2218232 INFO (qtp1576156728-17501) [n:127.0.0.1:43086_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n1] o.a.s.s.IndexSchema [halfdeletedcollection_shard1_replica_n1] Schema name=minimal
[junit4] 2> 2218234 INFO (qtp1576156728-17501) [n:127.0.0.1:43086_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2218234 INFO (qtp1576156728-17501) [n:127.0.0.1:43086_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'halfdeletedcollection_shard1_replica_n1' using configuration from collection halfdeletedcollection, trusted=true
[junit4] 2> 2218235 WARN (qtp1576156728-17501) [n:127.0.0.1:43086_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
[junit4] 2> 2218235 INFO (qtp1576156728-17501) [n:127.0.0.1:43086_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2218235 INFO (qtp1576156728-17501) [n:127.0.0.1:43086_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n1] o.a.s.c.SolrCore [[halfdeletedcollection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_B2503D85157AD516-001/tempDir-001/node2/halfdeletedcollection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_B2503D85157AD516-001/tempDir-001/node2/halfdeletedcollection_shard1_replica_n1/data/]
[junit4] 2> 2218266 INFO (qtp566194340-17518) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard2 r:core_node2 x:halfdeletedcollection_shard2_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2218266 INFO (qtp566194340-17518) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard2 r:core_node2 x:halfdeletedcollection_shard2_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2218267 INFO (qtp566194340-17518) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard2 r:core_node2 x:halfdeletedcollection_shard2_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2218267 INFO (qtp566194340-17518) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard2 r:core_node2 x:halfdeletedcollection_shard2_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2218268 INFO (qtp566194340-17518) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard2 r:core_node2 x:halfdeletedcollection_shard2_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@265cdca0[halfdeletedcollection_shard2_replica_n1] main]
[junit4] 2> 2218269 INFO (qtp566194340-17518) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard2 r:core_node2 x:halfdeletedcollection_shard2_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2218269 INFO (qtp566194340-17518) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard2 r:core_node2 x:halfdeletedcollection_shard2_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2218269 INFO (qtp1576156728-17501) [n:127.0.0.1:43086_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2218269 INFO (qtp1576156728-17501) [n:127.0.0.1:43086_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2218270 INFO (qtp566194340-17518) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard2 r:core_node2 x:halfdeletedcollection_shard2_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2218270 INFO (searcherExecutor-3957-thread-1-processing-n:127.0.0.1:37012_solr x:halfdeletedcollection_shard2_replica_n1 s:shard2 c:halfdeletedcollection r:core_node2) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard2 r:core_node2 x:halfdeletedcollection_shard2_replica_n1] o.a.s.c.SolrCore [halfdeletedcollection_shard2_replica_n1] Registered new searcher Searcher@265cdca0[halfdeletedcollection_shard2_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2218301 INFO (qtp566194340-17518) [n:127.0.0.1:37012_solr c:halfdeletedcollection s:shard2 r:core_node2 x:halfdeletedcollection_shard2_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1575799938010316800
[junit4] 2> 2218301 INFO (qtp1576156728-17501) [n:127.0.0.1:43086_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2218301 INFO (qtp1576156728-17501) [n:127.0.0.1:43086_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2218302 INFO (qtp1576156728-17501) [n:127.0.0.1:43086_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5b4dcbaa[halfdeletedcollection_shard1_replica_n1] main]
[junit4] 2> 2218303 INFO (qtp1576156728-17501) [n:127.0.0.1:43086_solr c:halfdeletedcollection s:shard1 r:core_node1 x:halfdeletedcollection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2218303 INFO
[...truncated too long message...]
iveMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> 4206360 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[B2503D85157AD516]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:49958 49958
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_B2503D85157AD516-001
[junit4] 2> Aug 15, 2017 12:58:36 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 19 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {_version_=PostingsFormat(name=Memory), id=PostingsFormat(name=LuceneFixedGap)}, docValues:{_version_=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=545, maxMBSortInHeap=5.746276723184316, sim=RandomSimilarity(queryNorm=false): {}, locale=ru-RU, timezone=Asia/Srednekolymsk
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=234255800,total=508559360
[junit4] 2> NOTE: All tests run in this JVM: [StatelessScriptUpdateProcessorFactoryTest, SuggesterTSTTest, BooleanFieldTest, TestHashPartitioner, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestSubQueryTransformerCrossCore, TestXIncludeConfig, JsonLoaderTest, TestTrieFacet, FileUtilsTest, HLLSerializationTest, DOMUtilTest, LoggingHandlerTest, DocValuesTest, BlockDirectoryTest, TestConfigReload, SpatialRPTFieldTypeTest, CdcrReplicationDistributedZkTest, TestQueryTypes, TestLRUCache, TestMaxTokenLenTokenizer, AsyncCallRequestStatusResponseTest, AtomicUpdatesTest, TestSlowCompositeReaderWrapper, LeaderInitiatedRecoveryOnCommitTest, TestMaxScoreQueryParser, AlternateDirectoryTest, TestFieldTypeCollectionResource, CacheHeaderTest, CoreAdminHandlerTest, TestFreeTextSuggestions, TolerantUpdateProcessorTest, SpatialFilterTest, TestPartialUpdateDeduplication, MetricUtilsTest, DisMaxRequestHandlerTest, TlogReplayBufferedWhileIndexingTest, TestConfigOverlay, HdfsWriteToMultipleCollectionsTest, TestLegacyNumericUtils, TestSha256AuthenticationProvider, ShardRoutingCustomTest, CSVRequestHandlerTest, TestCloudPivotFacet, TestAtomicUpdateErrorCases, TestPostingsSolrHighlighter, PrimUtilsTest, BadComponentTest, DateRangeFieldTest, TestApiFramework, TestSolrIndexConfig, TestDynamicLoading, OverseerModifyCollectionTest, ShufflingReplicaListTransformerTest, MoveReplicaTest, ZkStateReaderTest, UpdateRequestProcessorFactoryTest, TestSmileRequest, DeleteNodeTest, TestSegmentSorting, SolrCloudExampleTest, CollectionsAPIAsyncDistributedZkTest, RuleEngineTest, LargeFieldTest, TestCharFilters, DistributedQueryComponentCustomSortTest, TestRandomRequestDistribution, TestDynamicFieldResource, TestStressCloudBlindAtomicUpdates, TestGeoJSONResponseWriter, DocValuesMultiTest, SolrMetricsIntegrationTest, ConfigSetsAPITest, SolrCmdDistributorTest, HighlighterTest, CoreSorterTest, TestReloadDeadlock, TestHttpShardHandlerFactory, SOLR749Test, TestReqParamsAPI, SaslZkACLProviderTest, TestSchemaResource, TestFuzzyAnalyzedSuggestions, ZkCLITest, TestManagedSynonymFilterFactory, TestInPlaceUpdatesDistrib, BasicAuthIntegrationTest, TestClusterProperties, TestConfigSetsAPIExclusivity, OpenCloseCoreStressTest, AddBlockUpdateTest, TestCoreAdminApis, NoCacheHeaderTest, ChaosMonkeySafeLeaderTest, SolrJmxReporterCloudTest, TestRTGBase, AnalysisAfterCoreReloadTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestSolrXml, TestExactSharedStatsCache, SystemInfoHandlerTest, TestPerFieldSimilarity, TestManagedSchemaThreadSafety, IndexSchemaTest, SharedFSAutoReplicaFailoverUtilsTest, TestCursorMarkWithoutUniqueKey, DirectSolrSpellCheckerTest, TestCoreContainer, TestDeleteCollectionOnDownNodes, TestHalfAndHalfDocValues, TestAnalyzeInfixSuggestions, TestNumericTerms32, RequestLoggingTest, InfoHandlerTest, SoftAutoCommitTest, BasicDistributedZk2Test, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, LeaderElectionTest, TestDistributedGrouping, TestFaceting, ZkControllerTest, TestStressReorder, TestJoin, TestStressVersions, DistributedTermsComponentTest, SimpleFacetsTest, TestSolr4Spatial, SpellCheckComponentTest, TestGroupingSearch, PeerSyncTest, BadIndexSchemaTest, ConvertedLegacyTest, TestFiltering, BasicFunctionalityTest, DirectUpdateHandlerTest, TestBadConfig, ShowFileRequestHandlerTest, DistributedQueryElevationComponentTest, TestCoreDiscovery, TestExtendedDismaxParser, TestFoldingMultitermQuery, TestStressLucene, TestTrie, SuggesterWFSTTest, TestCSVLoader, PolyFieldTest, WordBreakSolrSpellCheckerTest, SchemaVersionSpecificBehaviorTest, SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, TestUpdate, FieldMutatingUpdateProcessorTest, QueryEqualityTest, DirectUpdateHandlerOptimizeTest, DefaultValueUpdateProcessorTest, DistanceFunctionTest, TestSolrDeletionPolicy1, DebugComponentTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, LukeRequestHandlerTest, TestQueryUtils, TestOmitPositions, FileBasedSpellCheckerTest, TermsComponentTest, TestValueSourceCache, PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, TestIndexingPerformance, MoreLikeThisHandlerTest, FieldAnalysisRequestHandlerTest, IndexSchemaRuntimeFieldTest, SolrPluginUtilsTest, TestJmxIntegration, ReturnFieldsTest, QueryParsingTest, TestAnalyzedSuggestions, SearchHandlerTest, UpdateParamsTest, CopyFieldTest, ResponseLogComponentTest, TestStressRecovery, TestMergePolicyConfig, TestSolrDeletionPolicy2, SampleTest, TestBinaryField, TestSearchPerf, MinimalSchemaTest, TestElisionMultitermQuery, TestConfig, OutputWriterTest, TestSolrCoreProperties, DirectSolrConnectionTest, TestCodecSupport, SynonymTokenizerTest, EchoParamsTest, TestSweetSpotSimilarityFactory, TestLMDirichletSimilarityFactory, TestIBSimilarityFactory, TestBM25SimilarityFactory, TestFastWriter, TimeZoneUtilsTest, ResourceLoaderTest, ScriptEngineTest, PluginInfoTest, TestFastLRUCache, DateMathParserTest, ChaosMonkeyNothingIsSafeTest, PreAnalyzedFieldTest, TestSystemIdResolver, TestSuggestSpellingConverter, DateFieldTest, SpellingQueryConverterTest, TestSolrJ, TestUtils, ZkNodePropsTest, SliceStateTest, UUIDFieldTest, CircularListTest, DistributedMLTComponentTest, CursorPagingTest, SolrTestCaseJ4Test, TestDistributedMissingSort, TestSimpleTrackingShardHandler, TestEmbeddedSolrServerSchemaAPI, TestJettySolrRunner, ActionThrottleTest, BaseCdcrDistributedZkTest, CdcrRequestHandlerTest, CleanupOldIndexTest, CollectionReloadTest, CollectionStateFormat2Test, CollectionsAPISolrJTest, ConnectionManagerTest, CustomCollectionTest, DeleteStatusTest, DistribDocExpirationUpdateProcessorTest, DocValuesNotIndexedTest, HttpPartitionTest, OverseerCollectionConfigSetProcessorTest, OverseerTaskQueueTest, RollingRestartTest, TestTolerantUpdateProcessorCloud, TestTolerantUpdateProcessorRandomCloud, TriLevelCompositeIdRoutingTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsChaosMonkeySafeLeaderTest, HdfsCollectionsAPIDistributedZkTest]
[junit4] Completed [561/729 (5!)] on J1 in 177.03s, 20 tests, 1 failure <<< FAILURES!
[...truncated 43906 lines...]
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 29 - Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/29/
8 tests failed.
FAILED: org.apache.lucene.index.TestDuelingCodecsAtNight.testBigEquals
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
at __randomizedtesting.SeedInfo.seed([634FA3D574374EDF:804BF850DF2C8A98]:0)
at org.apache.lucene.util.fst.BytesStore.skipBytes(BytesStore.java:307)
at org.apache.lucene.util.fst.FST.addNode(FST.java:701)
at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:126)
at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:200)
at org.apache.lucene.util.fst.Builder.finish(Builder.java:483)
at org.apache.lucene.codecs.memory.MemoryPostingsFormat$TermsWriter.finish(MemoryPostingsFormat.java:258)
at org.apache.lucene.codecs.memory.MemoryPostingsFormat$MemoryFieldsConsumer.write(MemoryPostingsFormat.java:391)
at org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:164)
at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:230)
at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:115)
at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4392)
at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4032)
at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2235)
at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5050)
at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1720)
at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1462)
at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:171)
at org.apache.lucene.index.TestDuelingCodecs.createRandomIndex(TestDuelingCodecs.java:139)
at org.apache.lucene.index.TestDuelingCodecsAtNight.testBigEquals(TestDuelingCodecsAtNight.java:34)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
FAILED: org.apache.lucene.search.TestInetAddressRangeQueries.testRandomBig
Error Message:
GC overhead limit exceeded
Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded
at __randomizedtesting.SeedInfo.seed([8F6A3C648F45D32C:83D41EB1E1CAFAC]:0)
at java.util.Arrays.copyOfRange(Arrays.java:3664)
at java.lang.String.<init>(String.java:207)
at java.lang.String.substring(String.java:1969)
at java.io.UnixFileSystem.normalize(UnixFileSystem.java:69)
at java.io.UnixFileSystem.normalize(UnixFileSystem.java:92)
at java.io.File.<init>(File.java:279)
at sun.security.provider.PolicyFile.canonPath(PolicyFile.java:1769)
at sun.security.provider.PolicyFile.canonicalizeCodebase(PolicyFile.java:1734)
at sun.security.provider.PolicyFile.access$700(PolicyFile.java:258)
at sun.security.provider.PolicyFile$5.run(PolicyFile.java:1188)
at sun.security.provider.PolicyFile$5.run(PolicyFile.java:1186)
at java.security.AccessController.doPrivileged(Native Method)
at sun.security.provider.PolicyFile.getPermissions(PolicyFile.java:1185)
at sun.security.provider.PolicyFile.getPermissions(PolicyFile.java:1132)
at sun.security.provider.PolicyFile.implies(PolicyFile.java:1086)
at java.security.ProtectionDomain.implies(ProtectionDomain.java:285)
at java.security.AccessControlContext.checkPermission(AccessControlContext.java:450)
at java.security.AccessController.checkPermission(AccessController.java:884)
at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
at sun.nio.fs.UnixChannelFactory.open(UnixChannelFactory.java:245)
at sun.nio.fs.UnixChannelFactory.newFileChannel(UnixChannelFactory.java:136)
at sun.nio.fs.UnixChannelFactory.newFileChannel(UnixChannelFactory.java:148)
at sun.nio.fs.UnixFileSystemProvider.newFileChannel(UnixFileSystemProvider.java:175)
at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
at org.apache.lucene.mockfile.DisableFsyncFS.newFileChannel(DisableFsyncFS.java:45)
at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:166)
at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:166)
at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:166)
at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
at java.nio.channels.FileChannel.open(FileChannel.java:287)
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([377BAFA2A1DF0C09:BF2F90780F2361F1]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:174)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
7 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest: 1) Thread[id=28690, name=zkCallback-2980-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=28689, name=zkCallback-2980-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=28691, name=zkCallback-2980-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=28615, name=StoppableCommitThread-SendThread(127.0.0.1:47664), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 5) Thread[id=28617, name=zkCallback-2980-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=28575, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55) 7) Thread[id=28616, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 7 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:
1) Thread[id=28690, name=zkCallback-2980-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=28689, name=zkCallback-2980-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=28691, name=zkCallback-2980-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=28615, name=StoppableCommitThread-SendThread(127.0.0.1:47664), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
5) Thread[id=28617, name=zkCallback-2980-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=28575, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55)
7) Thread[id=28616, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
at __randomizedtesting.SeedInfo.seed([377BAFA2A1DF0C09]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=28690, name=zkCallback-2980-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=28689, name=zkCallback-2980-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=28691, name=zkCallback-2980-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=28615, name=StoppableCommitThread-SendThread(127.0.0.1:47664), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 5) Thread[id=28617, name=zkCallback-2980-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=28690, name=zkCallback-2980-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=28689, name=zkCallback-2980-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=28691, name=zkCallback-2980-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=28615, name=StoppableCommitThread-SendThread(127.0.0.1:47664), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
5) Thread[id=28617, name=zkCallback-2980-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([377BAFA2A1DF0C09]:0)
FAILED: org.apache.solr.update.AutoCommitTest.testCommitWithin
Error Message:
Exception during query
Stack Trace:
java.lang.RuntimeException: Exception during query
at __randomizedtesting.SeedInfo.seed([377BAFA2A1DF0C09:8DA9C0DA22F1E21C]:0)
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:885)
at org.apache.solr.update.AutoCommitTest.testCommitWithin(AutoCommitTest.java:353)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
</response>
request was:q=id:529&qt=&start=0&rows=20&version=2.2
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:878)
... 40 more
FAILED: org.apache.solr.update.AutoCommitTest.testMaxDocs
Error Message:
Exception during query
Stack Trace:
java.lang.RuntimeException: Exception during query
at __randomizedtesting.SeedInfo.seed([377BAFA2A1DF0C09:8EFA797D8D350883]:0)
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:885)
at org.apache.solr.update.AutoCommitTest.testMaxDocs(AutoCommitTest.java:225)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
</response>
request was:q=id:14&qt=&start=0&rows=20&version=2.2
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:878)
... 40 more
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
Timeout waiting for all live and active
Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
at __randomizedtesting.SeedInfo.seed([377BAFA2A1DF0C09:BF2F90780F2361F1]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 1779 lines...]
[junit4] Suite: org.apache.lucene.index.TestDuelingCodecsAtNight
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestDuelingCodecsAtNight -Dtests.method=testBigEquals -Dtests.seed=634FA3D574374EDF -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=bg-BG -Dtests.timezone=CST6CDT -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 894s J2 | TestDuelingCodecsAtNight.testBigEquals <<<
[junit4] > Throwable #1: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([634FA3D574374EDF:804BF850DF2C8A98]:0)
[junit4] > at org.apache.lucene.util.fst.BytesStore.skipBytes(BytesStore.java:307)
[junit4] > at org.apache.lucene.util.fst.FST.addNode(FST.java:701)
[junit4] > at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:126)
[junit4] > at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:200)
[junit4] > at org.apache.lucene.util.fst.Builder.finish(Builder.java:483)
[junit4] > at org.apache.lucene.codecs.memory.MemoryPostingsFormat$TermsWriter.finish(MemoryPostingsFormat.java:258)
[junit4] > at org.apache.lucene.codecs.memory.MemoryPostingsFormat$MemoryFieldsConsumer.write(MemoryPostingsFormat.java:391)
[junit4] > at org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
[junit4] > at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:164)
[junit4] > at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:230)
[junit4] > at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:115)
[junit4] > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4392)
[junit4] > at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4032)
[junit4] > at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
[junit4] > at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2235)
[junit4] > at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5050)
[junit4] > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1720)
[junit4] > at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1462)
[junit4] > at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:171)
[junit4] > at org.apache.lucene.index.TestDuelingCodecs.createRandomIndex(TestDuelingCodecs.java:139)
[junit4] > at org.apache.lucene.index.TestDuelingCodecsAtNight.testBigEquals(TestDuelingCodecsAtNight.java:34)
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDuelingCodecsAtNight_634FA3D574374EDF-001
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true): {titleTokenized=IB SPL-D2, body=DFR GLZ(0.3)}, locale=bg-BG, timezone=CST6CDT
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=228846216,total=380633088
[junit4] 2> NOTE: All tests run in this JVM: [TestMathUtil, Test2BNumericDocValues, TestSpanSearchEquivalence, TestSearchForDuplicates, TestOneMergeWrappingMergePolicy, TestTwoPhaseCommitTool, TestSloppyPhraseQuery2, TestStressAdvance, TestSegmentReader, TestNumericUtils, TestMutablePointsReaderUtils, TestBlockPostingsFormat, TestBagOfPostings, TestTopDocsMerge, TestFixedBitSet, TestFieldReuse, TestIndexWriterReader, TestSpans, TestHighCompressionMode, TestStressIndexing2, TestRamUsageEstimator, TestLazyProxSkipping, TestIntsRef, TestLongPostings, TestIndexManyDocuments, TestPrefixQuery, TestDisjunctionMaxQuery, TestPayloadsOnVectors, TestLogMergePolicy, TestIntArrayDocIdSet, TestCompiledAutomaton, TestTermScorer, TestSortedSetDocValues, TestSimpleFSDirectory, TestFilterDirectory, TestRegexpQuery, TestSetOnce, TestTimSorterWorstCase, TestNRTReaderWithThreads, TestDateSort, Test2BBKDPoints, TestUsageTrackingFilterCachingPolicy, TestConjunctionDISI, TestSegmentTermEnum, TestCharsRefBuilder, TestTermVectors, TestHugeRamFile, TestVersion, Test2BPositions, TestMergePolicyWrapper, TestIndexSearcher, TestEarlyTerminatingSortingCollector, TestDirectMonotonic, TestBasics, TestDocValues, TestMultiFields, TestSameTokenSamePosition, TestSpanTermQuery, TestSpanBoostQuery, TestSortRescorer, TestCollectionUtil, TestMultiset, TestStopFilter, TestIndexWriterMaxDocs, TestAssertions, TestIndexWriterMerging, TestByteBlockPool, TestFieldCacheRewriteMethod, TestIOUtils, TestSimilarityBase, TestBoolean2ScorerSupplier, TestSearchWithThreads, TestField, TestFilterWeight, TestLongRangeFieldQueries, TestSentinelIntSet, TestBooleanSimilarity, TestPerFieldDocValuesFormat, TestPackedInts, TestGraphTokenizers, TestAddIndexes, TestShardSearching, TestIndexWriterExceptions, TestMultiMMap, TestSearcherManager, TestDeletionPolicy, TestIndexWriterMergePolicy, TestNRTThreads, TestPersistentSnapshotDeletionPolicy, TestAtomicUpdate, TestStandardAnalyzer, TestIndexWriterCommit, TestTimeLimitingCollector, TestSearchAfter, TestNeverDelete, TestStressIndexing, TestMinimize, TestRollingUpdates, TestIndexWriterUnicode, TestTieredMergePolicy, TestIndexWriterForceMerge, TestPerFieldPostingsFormat, TestArrayUtil, TestLevenshteinAutomata, TestByteSlices, TestBufferedIndexInput, TestTermsEnum, TestConsistentFieldNumbers, TestDirectoryReader, TestBagOfPositions, TestOmitNorms, TestComplexExplanations, TestRegexpRandom2, TestFastDecompressionMode, TestParallelCompositeReader, TestCrash, TestMultiThreadTermVectors, TestSegmentTermDocs, TestPerFieldPostingsFormat2, TestMultiDocValues, TestFieldsReader, TestSimpleSearchEquivalence, TestCustomSearcherSort, TestDocValuesIndexing, TestBooleanMinShouldMatch, TestDeterminizeLexicon, Test2BDocs, TestIndexWriterOnDiskFull, TestPostingsOffsets, TestMixedCodecs, TestTransactionRollback, TestFlex, TestUTF32ToUTF8, TestSpanExplanations, TestFastCompressionMode, TestDeterminism, TestMergeSchedulerExternal, TestTermVectorsWriter, TestCodecs, TestCharTermAttributeImpl, TestOmitPositions, TestBytesRefHash, TestNearSpansOrdered, TestTermRangeQuery, TestDocumentsWriterDeleteQueue, TestScorerPerf, TestMultiTermConstantScore, TestLiveFieldValues, TestPayloads, TestWildcard, TestSimilarity2, TestExternalCodecs, TestNoDeletionPolicy, TestSimpleExplanationsOfNonMatches, TestTermVectorsReader, TestIntBlockPool, TestTopDocsCollector, TestUniqueTermCount, TestBooleanQuery, TestComplexExplanationsOfNonMatches, TestUnicodeUtil, TestPrefixCodedTerms, TestAllFilesHaveCodecHeader, TestAutomatonQuery, TestDocument, TestMaxTermFrequency, TestFileSwitchDirectory, TestSumDocFreq, TestDocValuesScoring, TestWildcardRandom, TestRegexpRandom, TestMultiLevelSkipList, TestSubScorerFreqs, TestRecyclingIntBlockAllocator, TestIndexWriterConfig, TestConjunctions, TestSpanExplanationsOfNonMatches, TestSmallFloat, TestFilterIterator, TestCharsRef, TestDocCount, TestBinaryDocument, TestAutomatonQueryUnicode, TestAttributeSource, TestBooleanScorer, TestSpanFirstQuery, TestNoMergePolicy, TestIndexWriterLockRelease, TestBytesRef, TestDemo, TestNot, TestCloseableThreadLocal, MultiCollectorTest, TestNGramPhraseQuery, TestScoreCachingWrappingScorer, TestTermdocPerf, TestIndexCommit, TestTerm, Test2BPostingsBytes, TestPositiveScoresOnlyCollector, TestNewestSegment, TestByteArrayDataInput, Test2BBinaryDocValues, Test2BPagedBytes, TestCharacterUtils, TestDelegatingAnalyzerWrapper, TestReusableStringReader, TestWordlistLoader, TestPackedTokenAttributeImpl, TestBlockPostingsFormat3, TestForUtil, TestLucene50StoredFieldsFormat, TestLucene50StoredFieldsFormatHighCompression, TestLucene60PointsFormat, TestIndexedDISI, TestLucene70NormsFormat, TestDocIDMerger, TestDocsWithFieldSet, TestDuelingCodecsAtNight]
[junit4] Completed [454/454 (1!)] on J2 in 915.71s, 3 tests, 1 error <<< FAILURES!
[...truncated 1 lines...]
[junit4] JVM J2: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/core/test/temp/junit4-J2-20170814_043250_2391290353746117750839.sysout
[junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/heapdumps/java_pid6993.hprof ...
[junit4] Heap dump file created [424432249 bytes in 1.613 secs]
[junit4] <<< JVM J2: EOF ----
[...truncated 6407 lines...]
[junit4] Suite: org.apache.lucene.search.TestInetAddressRangeQueries
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestInetAddressRangeQueries -Dtests.method=testRandomBig -Dtests.seed=8F6A3C648F45D32C -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=be-BY -Dtests.timezone=America/Miquelon -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 514s J2 | TestInetAddressRangeQueries.testRandomBig <<<
[junit4] > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit exceeded
[junit4] > at __randomizedtesting.SeedInfo.seed([8F6A3C648F45D32C:83D41EB1E1CAFAC]:0)
[junit4] > at java.util.Arrays.copyOfRange(Arrays.java:3664)
[junit4] > at java.lang.String.<init>(String.java:207)
[junit4] > at java.lang.String.substring(String.java:1969)
[junit4] > at java.io.UnixFileSystem.normalize(UnixFileSystem.java:69)
[junit4] > at java.io.UnixFileSystem.normalize(UnixFileSystem.java:92)
[junit4] > at java.io.File.<init>(File.java:279)
[junit4] > at sun.security.provider.PolicyFile.canonPath(PolicyFile.java:1769)
[junit4] > at sun.security.provider.PolicyFile.canonicalizeCodebase(PolicyFile.java:1734)
[junit4] > at sun.security.provider.PolicyFile.access$700(PolicyFile.java:258)
[junit4] > at sun.security.provider.PolicyFile$5.run(PolicyFile.java:1188)
[junit4] > at sun.security.provider.PolicyFile$5.run(PolicyFile.java:1186)
[junit4] > at java.security.AccessController.doPrivileged(Native Method)
[junit4] > at sun.security.provider.PolicyFile.getPermissions(PolicyFile.java:1185)
[junit4] > at sun.security.provider.PolicyFile.getPermissions(PolicyFile.java:1132)
[junit4] > at sun.security.provider.PolicyFile.implies(PolicyFile.java:1086)
[junit4] > at java.security.ProtectionDomain.implies(ProtectionDomain.java:285)
[junit4] > at java.security.AccessControlContext.checkPermission(AccessControlContext.java:450)
[junit4] > at java.security.AccessController.checkPermission(AccessController.java:884)
[junit4] > at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
[junit4] > at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
[junit4] > at sun.nio.fs.UnixChannelFactory.open(UnixChannelFactory.java:245)
[junit4] > at sun.nio.fs.UnixChannelFactory.newFileChannel(UnixChannelFactory.java:136)
[junit4] > at sun.nio.fs.UnixChannelFactory.newFileChannel(UnixChannelFactory.java:148)
[junit4] > at sun.nio.fs.UnixFileSystemProvider.newFileChannel(UnixFileSystemProvider.java:175)
[junit4] > at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
[junit4] > at org.apache.lucene.mockfile.DisableFsyncFS.newFileChannel(DisableFsyncFS.java:45)
[junit4] > at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
[junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:166)
[junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:166)
[junit4] > at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:166)
[junit4] > at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:202)
[junit4] > at java.nio.channels.FileChannel.open(FileChannel.java:287)
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/misc/test/J2/temp/lucene.search.TestInetAddressRangeQueries_8F6A3C648F45D32C-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{ipRangeField=DocValuesFormat(name=Direct), id=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=842, maxMBSortInHeap=5.19664640246352, sim=RandomSimilarity(queryNorm=true): {}, locale=be-BY, timezone=America/Miquelon
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=178953232,total=260046848
[junit4] 2> NOTE: All tests run in this JVM: [TestDiversifiedTopDocsCollector, TestPKIndexSplitter, TestInetAddressPoint, TestLazyDocument, TestInetAddressRangeQueries]
[junit4] Completed [13/13 (1!)] on J2 in 532.64s, 4 tests, 1 error <<< FAILURES!
[...truncated 1 lines...]
[junit4] JVM J2: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/misc/test/temp/junit4-J2-20170814_064153_4411895428359212971211.sysout
[junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/heapdumps/java_pid11479.hprof ...
[junit4] Heap dump file created [577076422 bytes in 3.080 secs]
[junit4] <<< JVM J2: EOF ----
[...truncated 4249 lines...]
[junit4] Suite: org.apache.solr.update.AutoCommitTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J2/temp/solr.update.AutoCommitTest_377BAFA2A1DF0C09-001/init-core-data-001
[junit4] 2> 2206274 WARN (SUITE-AutoCommitTest-seed#[377BAFA2A1DF0C09]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
[junit4] 2> 2206274 INFO (SUITE-AutoCommitTest-seed#[377BAFA2A1DF0C09]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 2206276 INFO (SUITE-AutoCommitTest-seed#[377BAFA2A1DF0C09]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 2206276 INFO (SUITE-AutoCommitTest-seed#[377BAFA2A1DF0C09]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore
[junit4] 2> 2206277 INFO (SUITE-AutoCommitTest-seed#[377BAFA2A1DF0C09]-worker) [ ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 2206300 INFO (SUITE-AutoCommitTest-seed#[377BAFA2A1DF0C09]-worker) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2206318 INFO (SUITE-AutoCommitTest-seed#[377BAFA2A1DF0C09]-worker) [ ] o.a.s.s.IndexSchema [null] Schema name=test
[junit4] 2> 2206409 INFO (SUITE-AutoCommitTest-seed#[377BAFA2A1DF0C09]-worker) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2206441 INFO (SUITE-AutoCommitTest-seed#[377BAFA2A1DF0C09]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38afc06c
[junit4] 2> 2206445 INFO (SUITE-AutoCommitTest-seed#[377BAFA2A1DF0C09]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38afc06c
[junit4] 2> 2206445 INFO (SUITE-AutoCommitTest-seed#[377BAFA2A1DF0C09]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38afc06c
[junit4] 2> 2206446 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 2206468 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2206484 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2206571 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2206576 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1, trusted=true
[junit4] 2> 2206576 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38afc06c
[junit4] 2> 2206577 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2206577 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J2/temp/solr.update.AutoCommitTest_377BAFA2A1DF0C09-001/init-core-data-001/]
[junit4] 2> 2206578 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=71.568359375, floorSegmentMB=1.365234375, forceMergeDeletesPctAllowed=14.271025111947228, segmentsPerTier=21.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 2206613 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2206613 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2206614 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2206614 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2206615 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=62.6728515625, floorSegmentMB=2.001953125, forceMergeDeletesPctAllowed=28.34769281575448, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.480934025619307
[junit4] 2> 2206615 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7d1ba536[collection1] main]
[junit4] 2> 2206615 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 2206632 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 2206669 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2206669 INFO (searcherExecutor-6495-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 2206669 INFO (coreLoadExecutor-6494-thread-1) [ x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1575694280429666304
[junit4] 2> 2206670 INFO (searcherExecutor-6495-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 2206670 INFO (searcherExecutor-6495-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 2206670 INFO (searcherExecutor-6495-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 2206670 INFO (searcherExecutor-6495-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 2206671 INFO (searcherExecutor-6495-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 2206671 INFO (searcherExecutor-6495-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 2206671 INFO (searcherExecutor-6495-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 2206671 INFO (searcherExecutor-6495-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 2206672 INFO (SUITE-AutoCommitTest-seed#[377BAFA2A1DF0C09]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore end
[junit4] 2> 2206674 INFO (searcherExecutor-6495-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7d1ba536[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2206675 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testMaxTime
[junit4] 2> 2206675 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ ] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
[junit4] 2> 2206676 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@4e7972a9[collection1] realtime]
[junit4] 2> 2206676 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:* (-9223372036854775807)} 0 1
[junit4] 2> 2206677 ERROR (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ ] o.a.s.c.CorePropertiesLocator Couldn't load core descriptor from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/core.properties:java.nio.file.NoSuchFileException: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/core.properties
[junit4] 2> 2206678 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 2206701 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2206727 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ ] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2206801 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2206816 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ ] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1
[junit4] 2> 2206830 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38afc06c
[junit4] 2> 2206830 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J2/temp/solr.update.AutoCommitTest_377BAFA2A1DF0C09-001/init-core-data-001/]
[junit4] 2> 2206870 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2206870 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2206871 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@79941bc3[collection1] main]
[junit4] 2> 2206872 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 2206872 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 2206874 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2206875 INFO (searcherExecutor-6500-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 2206875 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1575694280645672960
[junit4] 2> 2206875 INFO (searcherExecutor-6500-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 2206875 INFO (searcherExecutor-6500-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 2206875 INFO (searcherExecutor-6500-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 2206876 INFO (searcherExecutor-6500-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 2206876 INFO (searcherExecutor-6500-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 2206876 INFO (searcherExecutor-6500-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 2206877 INFO (searcherExecutor-6500-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 2206878 INFO (searcherExecutor-6500-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 2206878 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@7dd6c0fd
[junit4] 2> 2206879 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 2206880 INFO (searcherExecutor-6500-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@79941bc3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2206880 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@74d36d7d[collection1] main]
[junit4] 2> 2206881 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@76afdb85
[junit4] 2> 2206881 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1991236485
[junit4] 2> 2206882 INFO (searcherExecutor-6500-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@74d36d7d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2206891 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[529 (1575694280661401600)]} 0 1
[junit4] 2> 2206891 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:529&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 2208391 INFO (commitScheduler-6503-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2208393 INFO (commitScheduler-6503-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2e0bbf49[collection1] main]
[junit4] 2> 2208393 INFO (commitScheduler-6503-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2208395 INFO (searcherExecutor-6500-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2e0bbf49[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):C1)))}
[junit4] 2> 2208395 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[530 (1575694282239508480)]} 0 1505
[junit4] 2> 2208396 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:529&qt=&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
[junit4] 2> 2208397 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:530&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 2208397 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{delete=[529 (-1575694282241605632)]} 0 0
[junit4] 2> 2208398 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:529&qt=&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
[junit4] 2> 2209895 INFO (commitScheduler-6503-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2209903 INFO (commitScheduler-6503-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5603027a[collection1] main]
[junit4] 2> 2209903 INFO (commitScheduler-6503-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2209904 INFO (searcherExecutor-6500-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5603027a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):C1)))}
[junit4] 2> 2210153 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[550 (1575694284081856512)]} 0 3262
[junit4] 2> 2210153 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:529&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 2210153 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1575694284082905088)]} 0 3263
[junit4] 2> 2210153 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1575694284082905089)]} 0 3263
[junit4] 2> 2210153 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1575694284082905090)]} 0 3263
[junit4] 2> 2210154 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1575694284082905091)]} 0 3263
[junit4] 2> 2210154 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1575694284083953664)]} 0 3263
[junit4] 2> 2210154 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1575694284083953665)]} 0 3263
[junit4] 2> 2210154 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1575694284083953666)]} 0 3264
[junit4] 2> 2210154 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1575694284083953667)]} 0 3264
[junit4] 2> 2210154 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1575694284083953668)]} 0 3264
[junit4] 2> 2210154 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1575694284083953669)]} 0 3264
[junit4] 2> 2210154 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:500&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 2211653 INFO (commitScheduler-6503-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2211654 INFO (commitScheduler-6503-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@73ed8db3[collection1] main]
[junit4] 2> 2211655 INFO (commitScheduler-6503-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2211656 INFO (searcherExecutor-6500-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@73ed8db3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):C1) Uninverting(_2(7.0.0):C11/9:delGen=1)))}
[junit4] 2> 2211657 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[531 (1575694285658914816)]} 0 4767
[junit4] 2> 2211657 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:500&qt=&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
[junit4] 2> 2211658 INFO (TEST-AutoCommitTest.testMaxTime-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testMaxTime
[junit4] 2> 2211659 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.SolrTestCaseJ4 ###Starting testCommitWithin
[junit4] 2> 2211659 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
[junit4] 2> 2211661 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4bc2a7c5[collection1] realtime]
[junit4] 2> 2211661 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:* (-9223372036854775807)} 0 1
[junit4] 2> 2211662 ERROR (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.CorePropertiesLocator Couldn't load core descriptor from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/core.properties:java.nio.file.NoSuchFileException: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/core.properties
[junit4] 2> 2211662 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 2211701 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2211721 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2211808 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2211815 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1
[junit4] 2> 2211826 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38afc06c
[junit4] 2> 2211826 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J2/temp/solr.update.AutoCommitTest_377BAFA2A1DF0C09-001/init-core-data-001/]
[junit4] 2> 2211876 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2211876 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2211877 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5448027d[collection1] main]
[junit4] 2> 2211878 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 2211879 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 2211881 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2211887 INFO (searcherExecutor-6504-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 2211887 INFO (searcherExecutor-6504-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 2211887 INFO (searcherExecutor-6504-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 2211889 INFO (searcherExecutor-6504-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 2211889 INFO (searcherExecutor-6504-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 2211889 INFO (searcherExecutor-6504-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 2211890 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=6.3125, floorSegmentMB=0.5224609375, forceMergeDeletesPctAllowed=29.53568060739504, segmentsPerTier=44.0, maxCFSSegmentSizeMB=1.2236328125, noCFSRatio=1.0
[junit4] 2> 2211890 INFO (searcherExecutor-6504-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 2211891 INFO (searcherExecutor-6504-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 2211891 INFO (searcherExecutor-6504-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 2211893 INFO (searcherExecutor-6504-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5448027d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2211893 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 2211893 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@707d3928[collection1] main]
[junit4] 2> 2211893 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@58c800b
[junit4] 2> 2211894 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=93093899
[junit4] 2> 2211897 INFO (searcherExecutor-6504-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@707d3928[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2211897 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[529 (1575694285910573056)]} 0 1
[junit4] 2> 2211897 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:529&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 2211898 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[530 (1575694285912670208)]} 0 2
[junit4] 2> 2211898 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:530&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 2211898 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[529 (-1575694285912670209)]} 0 2
[junit4] 2> 2211898 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:529&qt=&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 2211899 ERROR (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.SolrTestCaseJ4 REQUEST FAILED: xpath=//result[@numFound=1]
[junit4] 2> xml response was: <?xml version="1.0" encoding="UTF-8"?>
[junit4] 2> <response>
[junit4] 2> <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
[junit4] 2> </response>
[junit4] 2>
[junit4] 2> request was:q=id:529&qt=&start=0&rows=20&version=2.2
[junit4] 2> 2211899 ERROR (TEST-AutoCommitTest.testCommitWithin-seed#[377BAFA2A1DF0C09]) [ x:collection1] o.a.s.SolrTestCaseJ4 REQUEST FAILED: q=id:529&qt=&start=0&rows=20&version=2.2:java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
[junit4] 2> xml response was: <?xml version="1.0" encoding="UTF-8"?>
[junit4] 2> <response>
[junit4] 2> <lst name="responseHeader"><int name="status">0</int><
[...truncated too long message...]
rImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MethodMetric$2.snapshot(MethodMetric.java:111)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MethodMetric.snapshot(MethodMetric.java:144)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MetricsRegistry.snapshot(MetricsRegistry.java:401)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MetricsSourceBuilder$1.getMetrics(MetricsSourceBuilder.java:79)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
[junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
[junit4] 2> at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:204)
[junit4] 2> at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.hdfsFailoverAfterClass(SharedFSAutoReplicaFailoverTest.java:97)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_377BAFA2A1DF0C09-001
[junit4] 2> Aug 14, 2017 8:39:18 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 35 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=Direct), text=Lucene50(blocksize=128)}, docValues:{_version_=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=820, maxMBSortInHeap=6.499841783751084, sim=RandomSimilarity(queryNorm=true): {}, locale=ar, timezone=Asia/Irkutsk
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=274287728,total=522715136
[junit4] 2> NOTE: All tests run in this JVM: [TestManagedSynonymFilterFactory, TestPhraseSuggestions, AssignTest, TestNumericTerms32, ExternalFileFieldSortTest, TestSha256AuthenticationProvider, TestAuthenticationFramework, TestComponentsName, SolrMetricReporterTest, DeleteShardTest, TestUnifiedSolrHighlighter, TestIBSimilarityFactory, LeaderElectionIntegrationTest, SSLMigrationTest, HdfsUnloadDistributedZkTest, TestReloadDeadlock, HighlighterConfigTest, OpenExchangeRatesOrgProviderTest, SolrIndexConfigTest, SortSpecParsingTest, TestSmileRequest, TestGeoJSONResponseWriter, BooleanFieldTest, BlockJoinFacetDistribTest, PeerSyncTest, BasicDistributedZkTest, TestHighlightDedupGrouping, TestCloudRecovery, TestPayloadScoreQParserPlugin, AddBlockUpdateTest, AtomicUpdatesTest, TestReloadAndDeleteDocs, TestFunctionQuery, DistributedSuggestComponentTest, TestFaceting, TestDownShardTolerantSearch, SolrPluginUtilsTest, TestSolr4Spatial, TestInfoStreamLogging, TestQuerySenderNoQuery, TestMultiValuedNumericRangeQuery, TestDFISimilarityFactory, TestHashQParserPlugin, TestSolrCloudWithSecureImpersonation, DebugComponentTest, BinaryUpdateRequestHandlerTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, ExitableDirectoryReaderTest, TestBadConfig, TestTrieFacet, TestHalfAndHalfDocValues, TestBinaryResponseWriter, TestSchemaResource, PropertiesRequestHandlerTest, TestTolerantUpdateProcessorRandomCloud, TestFieldSortValues, TestSolrQueryParser, HLLSerializationTest, TestEmbeddedSolrServerConstructors, WrapperMergePolicyFactoryTest, DefaultValueUpdateProcessorTest, TestDocSet, DeleteLastCustomShardedReplicaTest, CursorPagingTest, MBeansHandlerTest, TestDistributedSearch, TestReplicaProperties, DistributedSpellCheckComponentTest, TestExactSharedStatsCache, TestStressLiveNodes, UUIDFieldTest, VersionInfoTest, TestPHPSerializedResponseWriter, SolrCloudReportersTest, TestRemoteStreaming, CloudExitableDirectoryReaderTest, SolrCLIZkUtilsTest, BlobRepositoryCloudTest, TestSolrCoreSnapshots, SynonymTokenizerTest, TestFieldCacheVsDocValues, DistributedFacetPivotWhiteBoxTest, TestMacros, CollectionTooManyReplicasTest, SharedFSAutoReplicaFailoverTest]
[junit4] Completed [696/728 (3!)] on J0 in 217.44s, 1 test, 1 failure <<< FAILURES!
[...truncated 7608 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/build.xml:818: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/build.xml:762: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid11479.hprof
* java_pid6993.hprof
Total time: 266 minutes 6 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 28 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/28/
6 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.analytics.NoFacetCloudTest
Error Message:
Could not find collection:collection1
Stack Trace:
java.lang.AssertionError: Could not find collection:collection1
at __randomizedtesting.SeedInfo.seed([C87F1642DCD87C78]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNotNull(Assert.java:526)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
at org.apache.solr.analytics.AbstractAnalyticsStatsCloudTest.setupCluster(AbstractAnalyticsStatsCloudTest.java:76)
at org.apache.solr.analytics.NoFacetCloudTest.populate(NoFacetCloudTest.java:62)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:847)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.StressHdfsTest
Error Message:
102 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.StressHdfsTest: 1) Thread[id=192392, name=qtp2106096125-192392, state=RUNNABLE, group=TGRP-StressHdfsTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=192354, name=qtp2041778238-192354, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=192508, name=zkCallback-27760-thread-4, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=192449, name=TEST-StressHdfsTest.test-seed#[7156DAFF8E4C6962]-EventThread, state=WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 5) Thread[id=192355, name=qtp2041778238-192355, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=192448, name=TEST-StressHdfsTest.test-seed#[7156DAFF8E4C6962]-SendThread(127.0.0.1:52251), state=TIMED_WAITING, group=TGRP-StressHdfsTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 7) Thread[id=192514, name=zkCallback-27740-thread-3, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=192403, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=192408, name=TEST-StressHdfsTest.test-seed#[7156DAFF8E4C6962]-SendThread(127.0.0.1:52251), state=TIMED_WAITING, group=TGRP-StressHdfsTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 10) Thread[id=192404, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=192503, name=OverseerHdfsCoreFailoverThread-98474728108851209-127.0.0.1:44879_-n_0000000001, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=192443, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=192410, name=zkCallback-27746-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=192373, name=zkCallback-27740-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=192357, name=qtp2041778238-192357, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=192391, name=qtp2106096125-192391, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=192411, name=Thread-87917, state=WAITING, group=TGRP-StressHdfsTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1592) 18) Thread[id=192444, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=192489, name=TEST-StressHdfsTest.test-seed#[7156DAFF8E4C6962]-EventThread, state=WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 20) Thread[id=192393, name=qtp2106096125-192393-acceptor-0@5e9810a5-ServerConnector@380baeb{HTTP/1.1,[http/1.1]}{127.0.0.1:35987}, state=RUNNABLE, group=TGRP-StressHdfsTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=192483, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=192488, name=TEST-StressHdfsTest.test-seed#[7156DAFF8E4C6962]-SendThread(127.0.0.1:52251), state=TIMED_WAITING, group=TGRP-StressHdfsTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 23) Thread[id=192494, name=watches-27761-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=192479, name=org.eclipse.jetty.server.session.HashSessionManager@341eb9edTimer, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=192484, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=192501, name=OverseerStateUpdate-98474728108851209-127.0.0.1:44879_-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:184) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=192387, name=searcherExecutor-13364-thread-1, state=WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 28) Thread[id=192464, name=searcherExecutor-13386-thread-1, state=WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=192510, name=zkCallback-27746-thread-2, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=192454, name=Scheduler-497806549, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 31) Thread[id=192379, name=watches-27741-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=192491, name=Thread-87948, state=WAITING, group=TGRP-StressHdfsTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1592) 33) Thread[id=192517, name=zkCallback-27740-thread-6, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 34) Thread[id=192515, name=zkCallback-27740-thread-4, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=192520, name=zkCallback-27753-thread-5, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=192430, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 37) Thread[id=192372, name=TEST-StressHdfsTest.test-seed#[7156DAFF8E4C6962]-EventThread, state=WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 38) Thread[id=192361, name=qtp2041778238-192361, state=RUNNABLE, group=TGRP-StressHdfsTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 39) Thread[id=192423, name=searcherExecutor-13375-thread-1, state=WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 40) Thread[id=192450, name=zkCallback-27753-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 41) Thread[id=192505, name=zkCallback-27753-thread-2, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 42) Thread[id=192362, name=org.eclipse.jetty.server.session.HashSessionManager@6e0319b4Timer, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 43) Thread[id=192425, name=updateExecutor-27743-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 44) Thread[id=192513, name=zkCallback-27746-thread-5, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 45) Thread[id=192433, name=qtp1273625196-192433-acceptor-0@76a722e9-ServerConnector@17803555{HTTP/1.1,[http/1.1]}{127.0.0.1:44994}, state=RUNNABLE, group=TGRP-StressHdfsTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 46) Thread[id=192356, name=qtp2041778238-192356-acceptor-0@740371fa-ServerConnector@4b39f2b{HTTP/1.1,[http/1.1]}{127.0.0.1:44879}, state=RUNNABLE, group=TGRP-StressHdfsTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 47) Thread[id=192439, name=org.eclipse.jetty.server.session.HashSessionManager@1e26d110Timer, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 48) Thread[id=192492, name=coreContainerWorkExecutor-13392-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 49) Thread[id=192471, name=qtp424389072-192471, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 50) Thread[id=192490, name=zkCallback-27760-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 51) Thread[id=192360, name=qtp2041778238-192360, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 52) Thread[id=192475, name=qtp424389072-192475, state=RUNNABLE, group=TGRP-StressHdfsTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 53) Thread[id=192395, name=qtp2106096125-192395, state=RUNNABLE, group=TGRP-StressHdfsTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 54) Thread[id=192409, name=TEST-StressHdfsTest.test-seed#[7156DAFF8E4C6962]-EventThread, state=WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 55) Thread[id=192477, name=qtp424389072-192477, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 56) Thread[id=192509, name=zkCallback-27760-thread-5, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 57) Thread[id=192470, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 58) Thread[id=192367, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 59) Thread[id=192412, name=coreContainerWorkExecutor-13370-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 60) Thread[id=192394, name=qtp2106096125-192394, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 61) Thread[id=192493, name=Scheduler-49146796, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 62) Thread[id=192455, name=watches-27754-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 63) Thread[id=192359, name=qtp2041778238-192359, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 64) Thread[id=192371, name=TEST-StressHdfsTest.test-seed#[7156DAFF8E4C6962]-SendThread(127.0.0.1:52251), state=TIMED_WAITING, group=TGRP-StressHdfsTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 65) Thread[id=192434, name=qtp1273625196-192434, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 66) Thread[id=192476, name=qtp424389072-192476, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 67) Thread[id=192366, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 68) Thread[id=192436, name=qtp1273625196-192436, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 69) Thread[id=192512, name=zkCallback-27746-thread-4, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 70) Thread[id=192376, name=coreContainerWorkExecutor-13359-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 71) Thread[id=192438, name=qtp1273625196-192438, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 72) Thread[id=192378, name=Scheduler-1982125458, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 73) Thread[id=192473, name=qtp424389072-192473-acceptor-0@535edb12-ServerConnector@549dce44{HTTP/1.1,[http/1.1]}{127.0.0.1:47754}, state=RUNNABLE, group=TGRP-StressHdfsTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 74) Thread[id=192437, name=qtp1273625196-192437, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 75) Thread[id=192465, name=updateExecutor-27750-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 76) Thread[id=192502, name=OverseerCollectionConfigSetProcessor-98474728108851209-127.0.0.1:44879_-n_0000000001, state=WAITING, group=Overseer collection creation process.] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210) at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357) at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 77) Thread[id=192466, name=recoveryExecutor-27751-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 78) Thread[id=192451, name=Thread-87933, state=WAITING, group=TGRP-StressHdfsTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1592) 79) Thread[id=192474, name=qtp424389072-192474, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 80) Thread[id=192478, name=qtp424389072-192478, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 81) Thread[id=192500, name=zkCallback-27740-thread-2, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 82) Thread[id=192390, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 83) Thread[id=192518, name=zkCallback-27753-thread-3, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 84) Thread[id=192507, name=zkCallback-27760-thread-3, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 85) Thread[id=192426, name=recoveryExecutor-27744-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 86) Thread[id=192396, name=qtp2106096125-192396, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 87) Thread[id=192472, name=qtp424389072-192472, state=RUNNABLE, group=TGRP-StressHdfsTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 88) Thread[id=192399, name=org.eclipse.jetty.server.session.HashSessionManager@1b3ff4dbTimer, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 89) Thread[id=192358, name=qtp2041778238-192358, state=RUNNABLE, group=TGRP-StressHdfsTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 90) Thread[id=192452, name=coreContainerWorkExecutor-13381-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 91) Thread[id=192431, name=qtp1273625196-192431, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 92) Thread[id=192415, name=watches-27747-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 93) Thread[id=192432, name=qtp1273625196-192432, state=RUNNABLE, group=TGRP-StressHdfsTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 94) Thread[id=192414, name=Scheduler-1909124043, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 95) Thread[id=192398, name=qtp2106096125-192398, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 96) Thread[id=192375, name=Thread-87902, state=WAITING, group=TGRP-StressHdfsTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1592) 97) Thread[id=192506, name=zkCallback-27760-thread-2, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 98) Thread[id=192511, name=zkCallback-27746-thread-3, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 99) Thread[id=192516, name=zkCallback-27740-thread-5, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 100) Thread[id=192397, name=qtp2106096125-192397, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 101) Thread[id=192519, name=zkCallback-27753-thread-4, state=TIMED_WAITING, group=TGRP-StressHdfsTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 102) Thread[id=192435, name=qtp1273625196-192435, state=RUNNABLE, group=TGRP-StressHdfsTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 102 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.StressHdfsTest:
1) Thread[id=192392, name=qtp2106096125-192392, state=RUNNABLE, group=TGRP-StressHdfsTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=192354, name=qtp2041778238-192354, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=192508, name=zkCallback-27760-thread-4, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=192449, name=TEST-StressHdfsTest.test-seed#[7156DAFF8E4C6962]-EventThread, state=WAITING, group=TGRP-StressHdfsTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
5) Thread[id=192355, name=qtp2041778238-192355, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=192448, name=TEST-StressHdfsTest.test-seed#[7156DAFF8E4C6962]-SendThread(127.0.0.1:52251), state=TIMED_WAITING, group=TGRP-StressHdfsTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
7) Thread[id=192514, name=zkCallback-27740-thread-3, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=192403, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=192408, name=TEST-StressHdfsTest.test-seed#[7156DAFF8E4C6962]-SendThread(127.0.0.1:52251), state=TIMED_WAITING, group=TGRP-StressHdfsTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
10) Thread[id=192404, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=192503, name=OverseerHdfsCoreFailoverThread-98474728108851209-127.0.0.1:44879_-n_0000000001, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=192443, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=192410, name=zkCallback-27746-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=192373, name=zkCallback-27740-thread-1, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=192357, name=qtp2041778238-192357, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
16) Thread[id=1
[...truncated too long message...]
java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 57) Thread[id=192475, name=qtp424389072-192475, state=RUNNABLE, group=TGRP-StressHdfsTest]
[junit4] > at sun.nio.ch.IOUtil.drain(Native Method)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 58) Thread[id=192519, name=zkCallback-27753-thread-4, state=TIMED_WAITING, group=TGRP-StressHdfsTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 59) Thread[id=192435, name=qtp1273625196-192435, state=RUNNABLE, group=TGRP-StressHdfsTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 60) Thread[id=192395, name=qtp2106096125-192395, state=RUNNABLE, group=TGRP-StressHdfsTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([7156DAFF8E4C6962]:0)
[junit4] Completed [559/728 (3!)] on J0 in 37.88s, 1 test, 3 errors <<< FAILURES!
[...truncated 506 lines...]
[junit4] JVM J2: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/temp/junit4-J2-20170813_051453_7097062345485626786408.sysout
[junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
[junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/heapdumps/java_pid2707.hprof ...
[junit4] Heap dump file created [565639218 bytes in 16.226 secs]
[junit4] <<< JVM J2: EOF ----
[junit4] JVM J2: stderr was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/temp/junit4-J2-20170813_051453_7098748095723811641090.syserr
[junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
[junit4]
[junit4] Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Connection evictor"
[junit4] <<< JVM J2: EOF ----
[...truncated 2629 lines...]
[junit4] Suite: org.apache.solr.analytics.NoFacetCloudTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/contrib/solr-analytics/test/J2/temp/solr.analytics.NoFacetCloudTest_C87F1642DCD87C78-001/init-core-data-001
[junit4] 2> Aug 13, 2017 6:25:21 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=false): {}, locale=hu-HU, timezone=America/Bogota
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=203589272,total=331350016
[junit4] 2> NOTE: All tests run in this JVM: [FieldFacetExtrasTest, NoFacetCloudTest]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=NoFacetCloudTest -Dtests.seed=C87F1642DCD87C78 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=hu-HU -Dtests.timezone=America/Bogota -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J2 | NoFacetCloudTest (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: Could not find collection:collection1
[junit4] > at __randomizedtesting.SeedInfo.seed([C87F1642DCD87C78]:0)
[junit4] > at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
[junit4] > at org.apache.solr.analytics.AbstractAnalyticsStatsCloudTest.setupCluster(AbstractAnalyticsStatsCloudTest.java:76)
[junit4] > at org.apache.solr.analytics.NoFacetCloudTest.populate(NoFacetCloudTest.java:62)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] Completed [16/17 (1!)] on J2 in 45.29s, 0 tests, 1 failure <<< FAILURES!
[...truncated 4898 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/build.xml:818: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/build.xml:762: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid2707.hprof
Total time: 253 minutes 45 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 27 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/27/
3 tests failed.
FAILED: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([3F40268D88935F6F:B7141957266F3297]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:222)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
Error Message:
18 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest: 1) Thread[id=22995, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 2) Thread[id=23231, name=zkCallback-2091-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=23278, name=zkCallback-2061-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=23132, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 5) Thread[id=22994, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]-SendThread(127.0.0.1:57744), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 6) Thread[id=23135, name=StoppableCommitThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1217) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:524) at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:544) at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:49) 7) Thread[id=23237, name=zkCallback-2061-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=23276, name=zkCallback-2091-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=23277, name=zkCallback-2061-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=23133, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 11) Thread[id=23279, name=zkCallback-2091-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=23131, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 13) Thread[id=23275, name=zkCallback-2061-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=23134, name=StoppableSearchThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210) at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357) at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354) at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1126) at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1112) at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:647) at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1227) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942) at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957) at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:54) 15) Thread[id=23227, name=StoppableIndexingThread-SendThread(127.0.0.1:57744), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230) at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1246) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1170) 16) Thread[id=22993, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=23280, name=zkCallback-2091-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=23228, name=StoppableIndexingThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 18 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest:
1) Thread[id=22995, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
2) Thread[id=23231, name=zkCallback-2091-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=23278, name=zkCallback-2061-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=23132, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
5) Thread[id=22994, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]-SendThread(127.0.0.1:57744), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
6) Thread[id=23135, name=StoppableCommitThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1217)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:524)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:544)
at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:49)
7) Thread[id=23237, name=zkCallback-2061-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=23276, name=zkCallback-2091-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=23277, name=zkCallback-2061-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=23133, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
11) Thread[id=23279, name=zkCallback-2091-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=23131, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
13) Thread[id=23275, name=zkCallback-2061-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=23134, name=StoppableSearchThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1126)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1112)
at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:647)
at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1227)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:54)
15) Thread[id=23227, name=StoppableIndexingThread-SendThread(127.0.0.1:57744), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230)
at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1246)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1170)
16) Thread[id=22993, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
17) Thread[id=23280, name=zkCallback-2091-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
18) Thread[id=23228, name=StoppableIndexingThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
at __randomizedtesting.SeedInfo.seed([3F40268D88935F6F]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=23231, name=zkCallback-2091-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=23278, name=zkCallback-2061-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=23132, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at org.apache.solr.common.util.JavaBinCodec.writePrimitive(JavaBinCodec.java:932) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91) at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104) at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195) at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115) 4) Thread[id=22994, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]-SendThread(127.0.0.1:57744), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 5) Thread[id=23237, name=zkCallback-2061-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=23276, name=zkCallback-2091-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=23277, name=zkCallback-2061-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=23133, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.io.PrintStream.println(PrintStream.java:805) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120) 9) Thread[id=23279, name=zkCallback-2091-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=23131, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:615) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91) at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104) at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195) at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115) 11) Thread[id=23275, name=zkCallback-2061-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=23134, name=StoppableSearchThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Throwable.printStackTrace(Throwable.java:643) at java.lang.Throwable.printStackTrace(Throwable.java:634) at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:57) 13) Thread[id=23227, name=StoppableIndexingThread-SendThread(127.0.0.1:57744), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 14) Thread[id=23280, name=zkCallback-2091-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=23231, name=zkCallback-2091-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=23278, name=zkCallback-2061-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=23132, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at org.apache.solr.common.util.JavaBinCodec.writePrimitive(JavaBinCodec.java:932)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
4) Thread[id=22994, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]-SendThread(127.0.0.1:57744), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
5) Thread[id=23237, name=zkCallback-2061-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=23276, name=zkCallback-2091-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=23277, name=zkCallback-2061-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=23133, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.io.PrintStream.println(PrintStream.java:805)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)
9) Thread[id=23279, name=zkCallback-2091-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=23131, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:615)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
11) Thread[id=23275, name=zkCallback-2061-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=23134, name=StoppableSearchThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Throwable.printStackTrace(Throwable.java:643)
at java.lang.Throwable.printStackTrace(Throwable.java:634)
at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:57)
13) Thread[id=23227, name=StoppableIndexingThread-SendThread(127.0.0.1:57744), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
14) Thread[id=23280, name=zkCallback-2091-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([3F40268D88935F6F]:0)
Build Log:
[...truncated 12691 lines...]
[junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_3F40268D88935F6F-001/init-core-data-001
[junit4] 2> 2440360 WARN (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[3F40268D88935F6F]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=26 numCloses=26
[junit4] 2> 2440360 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[3F40268D88935F6F]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 2440361 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[3F40268D88935F6F]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 2440361 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[3F40268D88935F6F]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /e/ei
[junit4] 2> 2440361 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.ChaosMonkeyNothingIsSafeWithPullReplicasTest Starting ChaosMonkey test with 2 shards and 4 nodes
[junit4] 2> 2440363 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2440364 INFO (Thread-11577) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2440364 INFO (Thread-11577) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2440365 ERROR (Thread-11577) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2440464 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.ZkTestServer start zk server on port:57744
[junit4] 2> 2440471 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 2440472 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 2440473 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 2440474 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 2440474 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 2440475 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 2440475 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 2440476 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 2440477 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 2440477 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 2440478 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 2440479 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
[junit4] 2> 2440566 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2440567 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4283e9e8{/e/ei,null,AVAILABLE}
[junit4] 2> 2440567 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@211dd0c9{HTTP/1.1,[http/1.1]}{127.0.0.1:52761}
[junit4] 2> 2440567 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.e.j.s.Server Started @2443899ms
[junit4] 2> 2440567 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_3F40268D88935F6F-001/tempDir-001/control/data, hostContext=/e/ei, hostPort=52761, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_3F40268D88935F6F-001/control-001/cores}
[junit4] 2> 2440567 ERROR (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2440567 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2440567 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2440567 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2440568 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-12T01:24:34.436Z
[junit4] 2> 2440570 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2440570 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_3F40268D88935F6F-001/control-001/solr.xml
[junit4] 2> 2440574 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5155e191, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2440576 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57744/solr
[junit4] 2> 2440677 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [n:127.0.0.1:52761_e%2Fei ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2440678 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [n:127.0.0.1:52761_e%2Fei ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:52761_e%2Fei
[junit4] 2> 2440678 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [n:127.0.0.1:52761_e%2Fei ] o.a.s.c.Overseer Overseer (id=98467910400868356-127.0.0.1:52761_e%2Fei-n_0000000000) starting
[junit4] 2> 2440699 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [n:127.0.0.1:52761_e%2Fei ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52761_e%2Fei
[junit4] 2> 2440700 INFO (zkCallback-2054-thread-1-processing-n:127.0.0.1:52761_e%2Fei) [n:127.0.0.1:52761_e%2Fei ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2440777 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [n:127.0.0.1:52761_e%2Fei ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5155e191
[junit4] 2> 2440781 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [n:127.0.0.1:52761_e%2Fei ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5155e191
[junit4] 2> 2440781 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [n:127.0.0.1:52761_e%2Fei ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5155e191
[junit4] 2> 2440782 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [n:127.0.0.1:52761_e%2Fei ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_3F40268D88935F6F-001/control-001/cores
[junit4] 2> 2440797 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2440797 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57744/solr ready
[junit4] 2> 2440798 INFO (qtp694828211-22959) [n:127.0.0.1:52761_e%2Fei ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:52761_e%252Fei&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2440799 INFO (OverseerThreadFactory-6711-thread-1-processing-n:127.0.0.1:52761_e%2Fei) [n:127.0.0.1:52761_e%2Fei ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 2440903 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2440904 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2441006 INFO (zkCallback-2054-thread-1-processing-n:127.0.0.1:52761_e%2Fei) [n:127.0.0.1:52761_e%2Fei ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2441918 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2441931 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 2442032 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 2442042 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 2442043 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5155e191
[junit4] 2> 2442043 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2442043 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_3F40268D88935F6F-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_3F40268D88935F6F-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 2442046 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@15f6df59
[junit4] 2> 2442052 WARN (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2442091 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2442091 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2442092 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 2442092 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2442093 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=65.4345703125, floorSegmentMB=0.5166015625, forceMergeDeletesPctAllowed=13.62740441356885, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 2442093 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3849658a[control_collection_shard1_replica_n1] main]
[junit4] 2> 2442094 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2442094 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2442095 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2442096 INFO (searcherExecutor-6714-thread-1-processing-n:127.0.0.1:52761_e%2Fei x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@3849658a[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2442096 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1575486568230027264
[junit4] 2> 2442100 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2442100 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2442101 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52761/e/ei/control_collection_shard1_replica_n1/
[junit4] 2> 2442101 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2442101 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:52761/e/ei/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 2442101 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2442103 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52761/e/ei/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 2442204 INFO (zkCallback-2054-thread-1-processing-n:127.0.0.1:52761_e%2Fei) [n:127.0.0.1:52761_e%2Fei ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2442206 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2442208 INFO (qtp694828211-22957) [n:127.0.0.1:52761_e%2Fei c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1304
[junit4] 2> 2442213 INFO (qtp694828211-22959) [n:127.0.0.1:52761_e%2Fei ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2442308 INFO (zkCallback-2054-thread-1-processing-n:127.0.0.1:52761_e%2Fei) [n:127.0.0.1:52761_e%2Fei ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2442800 INFO (OverseerCollectionConfigSetProcessor-98467910400868356-127.0.0.1:52761_e%2Fei-n_0000000000) [n:127.0.0.1:52761_e%2Fei ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2443213 INFO (qtp694828211-22959) [n:127.0.0.1:52761_e%2Fei ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:52761_e%252Fei&wt=javabin&version=2} status=0 QTime=2414
[junit4] 2> 2443221 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2443221 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57744/solr ready
[junit4] 2> 2443221 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 2443222 INFO (qtp694828211-22960) [n:127.0.0.1:52761_e%2Fei ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2443224 INFO (OverseerThreadFactory-6711-thread-2-processing-n:127.0.0.1:52761_e%2Fei) [n:127.0.0.1:52761_e%2Fei ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 2443226 WARN (OverseerThreadFactory-6711-thread-2-processing-n:127.0.0.1:52761_e%2Fei) [n:127.0.0.1:52761_e%2Fei ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 2443431 INFO (qtp694828211-22960) [n:127.0.0.1:52761_e%2Fei ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2443432 INFO (qtp694828211-22960) [n:127.0.0.1:52761_e%2Fei ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=209
[junit4] 2> 2443539 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_3F40268D88935F6F-001/shard-1-001 of type TLOG
[junit4] 2> 2443539 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2443540 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@223bc02{/e/ei,null,AVAILABLE}
[junit4] 2> 2443541 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@de7e30b{HTTP/1.1,[http/1.1]}{127.0.0.1:43961}
[junit4] 2> 2443541 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.e.j.s.Server Started @2446872ms
[junit4] 2> 2443541 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_3F40268D88935F6F-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/e/ei, hostPort=43961, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_3F40268D88935F6F-001/shard-1-001/cores}
[junit4] 2> 2443541 ERROR (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2443541 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2443541 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2443541 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2443541 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-12T01:24:37.409Z
[junit4] 2> 2443547 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2443547 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_3F40268D88935F6F-001/shard-1-001/solr.xml
[junit4] 2> 2443551 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5155e191, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2443553 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57744/solr
[junit4] 2> 2443561 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [n:127.0.0.1:43961_e%2Fei ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2443563 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [n:127.0.0.1:43961_e%2Fei ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2443564 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [n:127.0.0.1:43961_e%2Fei ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43961_e%2Fei
[junit4] 2> 2443565 INFO (zkCallback-2061-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2443565 INFO (zkCallback-2054-thread-1-processing-n:127.0.0.1:52761_e%2Fei) [n:127.0.0.1:52761_e%2Fei ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2443569 INFO (zkCallback-2066-thread-1-processing-n:127.0.0.1:43961_e%2Fei) [n:127.0.0.1:43961_e%2Fei ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2443608 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [n:127.0.0.1:43961_e%2Fei ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5155e191
[junit4] 2> 2443612 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [n:127.0.0.1:43961_e%2Fei ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5155e191
[junit4] 2> 2443612 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [n:127.0.0.1:43961_e%2Fei ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5155e191
[junit4] 2> 2443613 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[3F40268D88935F6F]) [n:127.0.0.1:43961_e%2Fei ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_3F40268D88935F6F-001/shard-1-001/cores
[junit4] 2> 2443630 INFO (qtp1937536246-23001) [n:127.0.0.1:43961_e%2Fei ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica w
[...truncated too long message...]
LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=23276, name=zkCallback-2091-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 7) Thread[id=23277, name=zkCallback-2061-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 8) Thread[id=23133, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
[junit4] > at java.io.PrintStream.println(PrintStream.java:805)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)
[junit4] > 9) Thread[id=23279, name=zkCallback-2091-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 10) Thread[id=23131, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:615)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
[junit4] > at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
[junit4] > at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
[junit4] > 11) Thread[id=23275, name=zkCallback-2061-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 12) Thread[id=23134, name=StoppableSearchThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
[junit4] > at java.lang.Throwable.printStackTrace(Throwable.java:643)
[junit4] > at java.lang.Throwable.printStackTrace(Throwable.java:634)
[junit4] > at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:57)
[junit4] > 13) Thread[id=23227, name=StoppableIndexingThread-SendThread(127.0.0.1:57744), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
[junit4] > 14) Thread[id=23280, name=zkCallback-2091-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([3F40268D88935F6F]:0)
[junit4] Completed [436/728 (1!)] on J1 in 171.02s, 1 test, 1 failure, 2 errors <<< FAILURES!
[...truncated 44265 lines...]
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 26 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/26/
4 tests failed.
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([D7DE1A0B47BC932A:5F8A25D1E940FED2]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:174)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest: 1) Thread[id=111129, name=zkCallback-30448-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=111056, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 3) Thread[id=111055, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 4) Thread[id=111128, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 5) Thread[id=111057, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55) 6) Thread[id=111171, name=zkCallback-30448-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=111127, name=StoppableCommitThread-SendThread(127.0.0.1:46514), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 8) Thread[id=111172, name=zkCallback-30448-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:
1) Thread[id=111129, name=zkCallback-30448-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=111056, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
3) Thread[id=111055, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
4) Thread[id=111128, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
5) Thread[id=111057, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55)
6) Thread[id=111171, name=zkCallback-30448-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=111127, name=StoppableCommitThread-SendThread(127.0.0.1:46514), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
8) Thread[id=111172, name=zkCallback-30448-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([D7DE1A0B47BC932A]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=111129, name=zkCallback-30448-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=111056, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.nio.cs.UTF_8$Encoder.encodeLoop(UTF_8.java:691) at java.nio.charset.CharsetEncoder.encode(CharsetEncoder.java:579) at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:271) at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125) at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:113) at java.io.OutputStreamWriter.write(OutputStreamWriter.java:194) at java.io.Writer.append(Writer.java:293) at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.beforeValue(JsonWriter.java:634) at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.open(JsonWriter.java:321) at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.beginObject(JsonWriter.java:304) at com.carrotsearch.ant.tasks.junit4.events.AppendStdErrEvent.serialize(AppendStdErrEvent.java:29) at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:119) at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114) at java.security.AccessController.doPrivileged(Native Method) at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114) at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99) at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472) at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) at java.io.PrintStream.flush(PrintStream.java:338) at java.io.FilterOutputStream.flush(FilterOutputStream.java:140) at java.io.PrintStream.write(PrintStream.java:482) at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221) at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291) at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104) at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185) at java.io.PrintStream.newLine(PrintStream.java:546) at java.io.PrintStream.println(PrintStream.java:807) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120) 3) Thread[id=111055, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.io.PrintStream.println(PrintStream.java:805) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120) 4) Thread[id=111171, name=zkCallback-30448-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=111127, name=StoppableCommitThread-SendThread(127.0.0.1:46514), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 6) Thread[id=111172, name=zkCallback-30448-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=111129, name=zkCallback-30448-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=111056, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.nio.cs.UTF_8$Encoder.encodeLoop(UTF_8.java:691)
at java.nio.charset.CharsetEncoder.encode(CharsetEncoder.java:579)
at sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:271)
at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:125)
at sun.nio.cs.StreamEncoder.write(StreamEncoder.java:113)
at java.io.OutputStreamWriter.write(OutputStreamWriter.java:194)
at java.io.Writer.append(Writer.java:293)
at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.beforeValue(JsonWriter.java:634)
at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.open(JsonWriter.java:321)
at com.carrotsearch.ant.tasks.junit4.gson.stream.JsonWriter.beginObject(JsonWriter.java:304)
at com.carrotsearch.ant.tasks.junit4.events.AppendStdErrEvent.serialize(AppendStdErrEvent.java:29)
at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:119)
at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114)
at java.security.AccessController.doPrivileged(Native Method)
at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472)
at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
at java.io.PrintStream.flush(PrintStream.java:338)
at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
at java.io.PrintStream.write(PrintStream.java:482)
at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)
at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)
at java.io.PrintStream.newLine(PrintStream.java:546)
at java.io.PrintStream.println(PrintStream.java:807)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)
3) Thread[id=111055, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.io.PrintStream.println(PrintStream.java:805)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)
4) Thread[id=111171, name=zkCallback-30448-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=111127, name=StoppableCommitThread-SendThread(127.0.0.1:46514), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
6) Thread[id=111172, name=zkCallback-30448-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([D7DE1A0B47BC932A]:0)
FAILED: org.apache.solr.core.HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig
Error Message:
The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~134.217728 MB.
Stack Trace:
java.lang.RuntimeException: The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~134.217728 MB.
at __randomizedtesting.SeedInfo.seed([D7DE1A0B47BC932A:2071D320BA357901]:0)
at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:311)
at org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:287)
at org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:227)
at org.apache.solr.core.HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig(HdfsDirectoryFactoryTest.java:111)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.OutOfMemoryError: Direct buffer memory
at java.nio.Bits.reserveMemory(Bits.java:694)
at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)
at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)
at org.apache.solr.store.blockcache.BlockCache.<init>(BlockCache.java:71)
at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:309)
... 42 more
Build Log:
[...truncated 11515 lines...]
[junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/init-core-data-001
[junit4] 2> 454344 WARN (SUITE-ChaosMonkeySafeLeaderWithPullReplicasTest-seed#[D7DE1A0B47BC932A]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=3 numCloses=3
[junit4] 2> 454347 INFO (SUITE-ChaosMonkeySafeLeaderWithPullReplicasTest-seed#[D7DE1A0B47BC932A]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 454370 INFO (SUITE-ChaosMonkeySafeLeaderWithPullReplicasTest-seed#[D7DE1A0B47BC932A]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 454378 INFO (SUITE-ChaosMonkeySafeLeaderWithPullReplicasTest-seed#[D7DE1A0B47BC932A]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 454378 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.ChaosMonkeySafeLeaderWithPullReplicasTest Starting ChaosMonkey test with 1 shards and 5 nodes
[junit4] 2> 454385 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 454394 INFO (Thread-125) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 454394 INFO (Thread-125) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 454422 ERROR (Thread-125) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 454497 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.ZkTestServer start zk server on port:46514
[junit4] 2> 454730 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 454733 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 454735 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 454736 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 454738 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 454739 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 454742 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 454746 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 454748 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 454759 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 454762 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 454774 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
[junit4] 2> 455049 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 455051 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@25a80d33{/,null,AVAILABLE}
[junit4] 2> 455067 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@76ef2762{SSL,[ssl, http/1.1]}{127.0.0.1:38209}
[junit4] 2> 455067 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.e.j.s.Server Started @459260ms
[junit4] 2> 455067 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/tempDir-001/control/data, replicaType=NRT, hostContext=/, hostPort=38209, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/control-001/cores}
[junit4] 2> 455068 ERROR (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 455068 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 455068 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 455068 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 455068 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-11T01:24:45.654Z
[junit4] 2> 455113 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 455113 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/control-001/solr.xml
[junit4] 2> 455124 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3839a70f, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 455128 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46514/solr
[junit4] 2> 455302 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:38209_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 455303 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:38209_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38209_
[junit4] 2> 455304 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:38209_ ] o.a.s.c.Overseer Overseer (id=98462248795570181-127.0.0.1:38209_-n_0000000000) starting
[junit4] 2> 455354 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:38209_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38209_
[junit4] 2> 455375 INFO (OverseerStateUpdate-98462248795570181-127.0.0.1:38209_-n_0000000000) [n:127.0.0.1:38209_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 455479 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:38209_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3839a70f
[junit4] 2> 455497 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:38209_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3839a70f
[junit4] 2> 455497 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:38209_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3839a70f
[junit4] 2> 455499 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:38209_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/control-001/cores
[junit4] 2> 455554 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 455638 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46514/solr ready
[junit4] 2> 456031 INFO (qtp565215463-110851) [n:127.0.0.1:38209_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38209_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 456047 INFO (OverseerThreadFactory-5734-thread-1-processing-n:127.0.0.1:38209_) [n:127.0.0.1:38209_ ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 456320 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 456320 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 456428 INFO (zkCallback-30408-thread-1-processing-n:127.0.0.1:38209_) [n:127.0.0.1:38209_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 457356 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 457375 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 457805 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 457860 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 457860 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3839a70f
[junit4] 2> 457861 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 457863 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 457868 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=5, maxMergeAtOnceExplicit=7, maxMergedSegmentMB=60.2353515625, floorSegmentMB=1.9345703125, forceMergeDeletesPctAllowed=15.800997917399041, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 457894 WARN (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 457986 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 457986 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 457988 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 457988 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 457990 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=49, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.62890625, noCFSRatio=0.0]
[junit4] 2> 457991 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1186faf2[control_collection_shard1_replica_n1] main]
[junit4] 2> 458001 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 458001 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 458002 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 458007 INFO (searcherExecutor-5737-thread-1-processing-n:127.0.0.1:38209_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@1186faf2[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 458008 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1575395984507142144
[junit4] 2> 458034 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 458034 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 458034 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:38209/control_collection_shard1_replica_n1/
[junit4] 2> 458034 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 458034 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:38209/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 458034 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 458053 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:38209/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 458164 INFO (zkCallback-30408-thread-1-processing-n:127.0.0.1:38209_) [n:127.0.0.1:38209_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 458211 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 458214 INFO (qtp565215463-110850) [n:127.0.0.1:38209_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1894
[junit4] 2> 458226 INFO (qtp565215463-110851) [n:127.0.0.1:38209_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 458329 INFO (zkCallback-30408-thread-2-processing-n:127.0.0.1:38209_) [n:127.0.0.1:38209_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 459228 INFO (qtp565215463-110851) [n:127.0.0.1:38209_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38209_&wt=javabin&version=2} status=0 QTime=3196
[junit4] 2> 459250 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 459251 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46514/solr ready
[junit4] 2> 459251 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 459266 INFO (qtp565215463-110848) [n:127.0.0.1:38209_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 459272 INFO (OverseerThreadFactory-5734-thread-2-processing-n:127.0.0.1:38209_) [n:127.0.0.1:38209_ ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 459273 INFO (OverseerCollectionConfigSetProcessor-98462248795570181-127.0.0.1:38209_-n_0000000000) [n:127.0.0.1:38209_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 459274 WARN (OverseerThreadFactory-5734-thread-2-processing-n:127.0.0.1:38209_) [n:127.0.0.1:38209_ ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 459485 INFO (qtp565215463-110848) [n:127.0.0.1:38209_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 459485 INFO (qtp565215463-110848) [n:127.0.0.1:38209_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=218
[junit4] 2> 459996 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/shard-1-001 of type TLOG
[junit4] 2> 459998 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 460000 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5e24c475{/,null,AVAILABLE}
[junit4] 2> 460001 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1e19f74{SSL,[ssl, http/1.1]}{127.0.0.1:51630}
[junit4] 2> 460001 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.e.j.s.Server Started @464195ms
[junit4] 2> 460001 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=51630, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/shard-1-001/cores}
[junit4] 2> 460002 ERROR (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 460002 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 460002 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 460002 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 460002 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-11T01:24:50.588Z
[junit4] 2> 460009 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 460009 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/shard-1-001/solr.xml
[junit4] 2> 460016 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3839a70f, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 460026 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46514/solr
[junit4] 2> 460057 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:51630_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 460060 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:51630_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 460062 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:51630_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51630_
[junit4] 2> 460063 INFO (zkCallback-30408-thread-2-processing-n:127.0.0.1:38209_) [n:127.0.0.1:38209_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 460068 INFO (zkCallback-30415-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 460068 INFO (zkCallback-30420-thread-1-processing-n:127.0.0.1:51630_) [n:127.0.0.1:51630_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 460212 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:51630_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3839a70f
[junit4] 2> 460221 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:51630_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3839a70f
[junit4] 2> 460221 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:51630_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3839a70f
[junit4] 2> 460224 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:51630_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/shard-1-001/cores
[junit4] 2> 460260 INFO (qtp565215463-110845) [n:127.0.0.1:38209_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:51630_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 460264 INFO (OverseerCollectionConfigSetProcessor-98462248795570181-127.0.0.1:38209_-n_0000000000) [n:127.0.0.1:38209_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 460266 INFO (OverseerThreadFactory-5734-thread-3-processing-n:127.0.0.1:38209_) [n:127.0.0.1:38209_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:51630_ for creating new replica
[junit4] 2> 460395 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t0&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 460396 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 461461 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 461494 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.s.IndexSchema [collection1_shard1_replica_t0] Schema name=test
[junit4] 2> 461722 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 461802 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t0' using configuration from collection collection1, trusted=true
[junit4] 2> 461806 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t0' (registry 'solr.core.collection1.shard1.replica_t0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3839a70f
[junit4] 2> 461813 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 461813 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SolrCore [[collection1_shard1_replica_t0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/shard-1-001/cores/collection1_shard1_replica_t0], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/shard-1-001/cores/collection1_shard1_replica_t0/data/]
[junit4] 2> 461822 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=5, maxMergeAtOnceExplicit=7, maxMergedSegmentMB=60.2353515625, floorSegmentMB=1.9345703125, forceMergeDeletesPctAllowed=15.800997917399041, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 461855 WARN (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 462003 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 462003 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 462005 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 462005 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 462007 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=49, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.62890625, noCFSRatio=0.0]
[junit4] 2> 462008 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.s.SolrIndexSearcher Opening [Searcher@2a3133f[collection1_shard1_replica_t0] main]
[junit4] 2> 462014 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 462015 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 462022 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 462025 INFO (searcherExecutor-5748-thread-1-processing-n:127.0.0.1:51630_ x:collection1_shard1_replica_t0 s:shard1 c:collection1) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SolrCore [collection1_shard1_replica_t0] Registered new searcher Searcher@2a3133f[collection1_shard1_replica_t0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 462025 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1575395988719271936
[junit4] 2> 462056 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 462056 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 462057 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:51630/collection1_shard1_replica_t0/
[junit4] 2> 462057 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 462057 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SyncStrategy https://127.0.0.1:51630/collection1_shard1_replica_t0/ has no replicas
[junit4] 2> 462057 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 462062 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.ZkController collection1_shard1_replica_t0 stopping background replication from leader
[junit4] 2> 462086 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:51630/collection1_shard1_replica_t0/ shard1
[junit4] 2> 462188 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 462191 INFO (qtp1768783487-110893) [n:127.0.0.1:51630_ c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t0&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1796
[junit4] 2> 462199 INFO (qtp565215463-110845) [n:127.0.0.1:38209_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:51630_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=1938
[junit4] 2> 462266 INFO (OverseerCollectionConfigSetProcessor-98462248795570181-127.0.0.1:38209_-n_0000000000) [n:127.0.0.1:38209_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 462448 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/shard-2-001 of type TLOG
[junit4] 2> 462456 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 462458 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@a841dfe{/,null,AVAILABLE}
[junit4] 2> 462458 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@16096185{SSL,[ssl, http/1.1]}{127.0.0.1:46712}
[junit4] 2> 462458 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.e.j.s.Server Started @466651ms
[junit4] 2> 462458 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=46712, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/shard-2-001/cores}
[junit4] 2> 462459 ERROR (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 462460 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 462460 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 462460 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 462460 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-11T01:24:53.046Z
[junit4] 2> 462465 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15dcee4c33f000b, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 462465 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 462465 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/shard-2-001/solr.xml
[junit4] 2> 462484 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3839a70f, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 462502 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46514/solr
[junit4] 2> 462576 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:46712_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 462581 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:46712_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 462586 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:46712_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46712_
[junit4] 2> 462588 INFO (zkCallback-30420-thread-1-processing-n:127.0.0.1:51630_) [n:127.0.0.1:51630_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 462588 INFO (zkCallback-30415-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 462589 INFO (zkCallback-30408-thread-2-processing-n:127.0.0.1:38209_) [n:127.0.0.1:38209_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 462604 INFO (zkCallback-30426-thread-1-processing-n:127.0.0.1:46712_) [n:127.0.0.1:46712_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 462683 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:46712_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3839a70f
[junit4] 2> 462739 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:46712_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3839a70f
[junit4] 2> 462740 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:46712_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3839a70f
[junit4] 2> 462742 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[D7DE1A0B47BC932A]) [n:127.0.0.1:46712_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/shard-2-001/cores
[junit4] 2> 462809 INFO (qtp565215463-110846) [n:127.0.0.1:38209_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:46712_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 462827 INFO (OverseerThreadFactory-5734-thread-4-processing-n:127.0.0.1:38209_) [n:127.0.0.1:38209_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:46712_ for creating new replica
[junit4] 2> 462904 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t1&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 462904 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 463927 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 463943 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.s.IndexSchema [collection1_shard1_replica_t1] Schema name=test
[junit4] 2> 464126 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 464199 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t1' using configuration from collection collection1, trusted=true
[junit4] 2> 464202 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t1' (registry 'solr.core.collection1.shard1.replica_t1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3839a70f
[junit4] 2> 464203 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 464203 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore [[collection1_shard1_replica_t1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/shard-2-001/cores/collection1_shard1_replica_t1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_D7DE1A0B47BC932A-001/shard-2-001/cores/collection1_shard1_replica_t1/data/]
[junit4] 2> 464207 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=5, maxMergeAtOnceExplicit=7, maxMergedSegmentMB=60.2353515625, floorSegmentMB=1.9345703125, forceMergeDeletesPctAllowed=15.800997917399041, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 464230 WARN (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 464317 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 464317 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 464318 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 464318 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 464320 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=49, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.62890625, noCFSRatio=0.0]
[junit4] 2> 464321 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7ae6cd7[collection1_shard1_replica_t1] main]
[junit4] 2> 464323 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 464323 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 464324 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 464325 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1575395991130996736
[junit4] 2> 464331 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.c.ZkController Core needs to recover:collection1_shard1_replica_t1
[junit4] 2> 464332 INFO (qtp129494907-110921) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t1&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1428
[junit4] 2> 464346 INFO (searcherExecutor-5759-thread-1-processing-n:127.0.0.1:46712_ x:collection1_shard1_replica_t1 s:shard1 c:collection1) [n:127.0.0.1:46712_ c:collection1 s:shard1 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore [collection1_shard1_replica_t1] Registered new searcher Searcher@7ae6cd7[collection1_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 464369 INFO (updateExecutor-30423-thread-1-processing-n:127.0.0.1:46712_ x:collection1_shard1_replica_t1 s:shard1 c:collection1) [n:127.0.0.1:46712_ c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 464369 INFO (recoveryExecutor-30424-thread-1-processing-n:127.0.0.1:46712_ x:collection1_shard1_replica_t1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:46712_ c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 464369 INFO (recoveryExecutor-30424-thread-1-processing-n:127.0.0.1:46712_ x:collection1_shard1_replica_t1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:46712_ c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 464369 INFO (recoveryExecutor-30424-thread-1-processing-n:127.0.0.1:46712_ x:collection1_shard1_replica_t1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:46712_ c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.c.ZkController collection1_shard1_replica_t1 stopping background replication from leader
[junit4] 2> 464370 INFO (recoveryExecutor-30424-thread-1-processing-n:127.0.0.1:46712_ x:collection1_shard1_replica_t1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:46712_ c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard1_replica_t1]
[junit4] 2> 464370 INFO (recoveryExecutor-30424-thread-1-processing-n:127.0.0.1:46712_ x:collection1_shard1_replica_t1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:46712_ c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 464370 INFO (recoveryExecutor-30424-thread-1-processing-n:127.0.0.1:46712_ x:collection1_shard1_replica_t1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:46712_ c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard1_replica_t1] as recovering, leader is [https://127.0.0.1:51630/collection1_shard1_replica_t0/] and I am [https://127.0.0.1:46712/collection1_shard1_replica_t1/]
[junit4] 2> 464377 INFO (qtp565215463-110846) [n:127.0.0.1:38209_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:46712_&action=ADDRE
[...truncated too long message...]
toryTest_D7DE1A0B47BC932A-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:46466) [ ] o.a.h.h.s.d.DataNode BPOfferService for Block pool BP-798218925-10.41.0.5-1502415398192 (Datanode Uuid e82ac448-bd63-479d-829e-eadaf3d9b2e8) service to localhost/127.0.0.1:46466 interrupted
[junit4] 2> 1171448 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_D7DE1A0B47BC932A-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_D7DE1A0B47BC932A-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:46466) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-798218925-10.41.0.5-1502415398192 (Datanode Uuid e82ac448-bd63-479d-829e-eadaf3d9b2e8) service to localhost/127.0.0.1:46466
[junit4] 2> 1171460 INFO (SUITE-HdfsDirectoryFactoryTest-seed#[D7DE1A0B47BC932A]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 1171568 ERROR (SUITE-HdfsDirectoryFactoryTest-seed#[D7DE1A0B47BC932A]-worker) [ ] o.a.h.m.l.MethodMetric Error invoking method getBlocksTotal
[junit4] 2> java.lang.reflect.InvocationTargetException
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MethodMetric$2.snapshot(MethodMetric.java:111)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MethodMetric.snapshot(MethodMetric.java:144)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MetricsRegistry.snapshot(MetricsRegistry.java:401)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MetricsSourceBuilder$1.getMetrics(MetricsSourceBuilder.java:79)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
[junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
[junit4] 2> at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:204)
[junit4] 2> at org.apache.solr.core.HdfsDirectoryFactoryTest.teardownClass(HdfsDirectoryFactoryTest.java:64)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_D7DE1A0B47BC932A-001
[junit4] 2> Aug 11, 2017 1:36:42 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=866, maxMBSortInHeap=5.556927223347858, sim=RandomSimilarity(queryNorm=true): {}, locale=es-AR, timezone=America/Bahia
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=310446480,total=520093696
[junit4] 2> NOTE: All tests run in this JVM: [TestTlogReplica, SecurityConfHandlerTest, CollectionsAPIAsyncDistributedZkTest, ParsingFieldUpdateProcessorsTest, TestSearchPerf, CollectionsAPIDistributedZkTest, TestHighFrequencyDictionaryFactory, TestLRUCache, TestManagedSynonymGraphFilterFactory, TlogReplayBufferedWhileIndexingTest, AutoScalingHandlerTest, PreAnalyzedFieldManagedSchemaCloudTest, JSONWriterTest, MoreLikeThisHandlerTest, CheckHdfsIndexTest, TestAtomicUpdateErrorCases, TestDistributedGrouping, TestCoreDiscovery, SchemaVersionSpecificBehaviorTest, MergeStrategyTest, TestPerFieldSimilarity, SuggesterTSTTest, LoggingHandlerTest, DocValuesMultiTest, TestFieldCacheSort, TestConfigOverlay, HdfsThreadLeakTest, TestLockTree, AnalyticsQueryTest, TestSystemIdResolver, DocValuesNotIndexedTest, TestValueSourceCache, TestTestInjection, TestConfigSetsAPIExclusivity, HdfsRecoveryZkTest, TestExceedMaxTermLength, HdfsBasicDistributedZkTest, TestDefaultStatsCache, MissingSegmentRecoveryTest, TestLegacyNumericUtils, OverseerTaskQueueTest, UpdateRequestProcessorFactoryTest, TestComplexPhraseQParserPlugin, FileUtilsTest, RequestLoggingTest, TestPartialUpdateDeduplication, TestClusterProperties, TestPKIAuthenticationPlugin, LeaderElectionContextKeyTest, DeleteNodeTest, HdfsDirectoryFactoryTest]
[junit4] Completed [583/728 (2!)] on J1 in 13.98s, 3 tests, 1 error <<< FAILURES!
[...truncated 43771 lines...]
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 25 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/25/
3 tests failed.
FAILED: org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test
Error Message:
Error from server at http://127.0.0.1:55730/m_/r: ADDREPLICA failed to create replica
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:55730/m_/r: ADDREPLICA failed to create replica
at __randomizedtesting.SeedInfo.seed([177FA75A9BC052C6:9F2B9880353C3F3E]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.UnloadDistributedZkTest.testUnloadShardAndCollection(UnloadDistributedZkTest.java:124)
at org.apache.solr.cloud.UnloadDistributedZkTest.test(UnloadDistributedZkTest.java:71)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.core.TestJmxIntegration.testJmxRegistration
Error Message:
org.apache.lucene.store.AlreadyClosedException: this Directory is closed
Stack Trace:
javax.management.RuntimeMBeanException: org.apache.lucene.store.AlreadyClosedException: this Directory is closed
at __randomizedtesting.SeedInfo.seed([177FA75A9BC052C6:99AEC360F6810AA3]:0)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.rethrow(DefaultMBeanServerInterceptor.java:839)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.rethrowMaybeMBeanException(DefaultMBeanServerInterceptor.java:852)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getAttribute(DefaultMBeanServerInterceptor.java:651)
at com.sun.jmx.mbeanserver.JmxMBeanServer.getAttribute(JmxMBeanServer.java:678)
at org.apache.solr.core.TestJmxIntegration.testJmxRegistration(TestJmxIntegration.java:121)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.lucene.store.AlreadyClosedException: this Directory is closed
at org.apache.lucene.store.BaseDirectory.ensureOpen(BaseDirectory.java:51)
at org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:235)
at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:352)
at org.apache.solr.core.SolrCore.getIndexDir(SolrCore.java:328)
at org.apache.solr.core.SolrCore.getIndexSize(SolrCore.java:405)
at org.apache.solr.core.SolrCore.lambda$initializeMetrics$8(SolrCore.java:1138)
at com.codahale.metrics.JmxReporter$JmxGauge.getValue(JmxReporter.java:188)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at sun.reflect.misc.Trampoline.invoke(MethodUtil.java:71)
at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at sun.reflect.misc.MethodUtil.invoke(MethodUtil.java:275)
at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:112)
at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:46)
at com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:237)
at com.sun.jmx.mbeanserver.PerInterface.getAttribute(PerInterface.java:83)
at com.sun.jmx.mbeanserver.MBeanSupport.getAttribute(MBeanSupport.java:206)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getAttribute(DefaultMBeanServerInterceptor.java:647)
... 41 more
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
Timeout waiting for all live and active
Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
at __randomizedtesting.SeedInfo.seed([177FA75A9BC052C6:9F2B9880353C3F3E]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12001 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001/init-core-data-001
[junit4] 2> 1518391 WARN (SUITE-HdfsUnloadDistributedZkTest-seed#[177FA75A9BC052C6]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=43 numCloses=43
[junit4] 2> 1518391 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[177FA75A9BC052C6]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 1518392 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[177FA75A9BC052C6]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 1518393 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[177FA75A9BC052C6]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /m_/r
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 1518441 WARN (SUITE-HdfsUnloadDistributedZkTest-seed#[177FA75A9BC052C6]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 1518456 WARN (SUITE-HdfsUnloadDistributedZkTest-seed#[177FA75A9BC052C6]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1518465 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[177FA75A9BC052C6]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 1518478 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[177FA75A9BC052C6]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_35657_hdfs____bfjejk/webapp
[junit4] 2> 1518849 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[177FA75A9BC052C6]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35657
[junit4] 2> 1518951 WARN (SUITE-HdfsUnloadDistributedZkTest-seed#[177FA75A9BC052C6]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1518952 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[177FA75A9BC052C6]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 1518965 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[177FA75A9BC052C6]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_60456_datanode____foi62q/webapp
[junit4] 2> 1519341 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[177FA75A9BC052C6]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:60456
[junit4] 2> 1519744 WARN (SUITE-HdfsUnloadDistributedZkTest-seed#[177FA75A9BC052C6]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1519746 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[177FA75A9BC052C6]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 1519804 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[177FA75A9BC052C6]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_36825_datanode____rjh82n/webapp
[junit4] 2> 1520097 INFO (IPC Server handler 2 on 40454) [ ] BlockStateChange BLOCK* processReport: from storage DS-9b8d8cc1-1c67-4296-93ce-c70c85c0e635 node DatanodeRegistration(127.0.0.1:53909, datanodeUuid=a4eecaa7-ac59-49ad-a97d-229a225bd836, infoPort=42652, infoSecurePort=0, ipcPort=42675, storageInfo=lv=-56;cid=testClusterID;nsid=949490773;c=0), blocks: 0, hasStaleStorage: true, processing time: 9 msecs
[junit4] 2> 1520097 INFO (IPC Server handler 2 on 40454) [ ] BlockStateChange BLOCK* processReport: from storage DS-9f61f565-dda2-49d1-8724-7c4f4e0c12c5 node DatanodeRegistration(127.0.0.1:53909, datanodeUuid=a4eecaa7-ac59-49ad-a97d-229a225bd836, infoPort=42652, infoSecurePort=0, ipcPort=42675, storageInfo=lv=-56;cid=testClusterID;nsid=949490773;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 1520583 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[177FA75A9BC052C6]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36825
[junit4] 2> 1521151 INFO (IPC Server handler 5 on 40454) [ ] BlockStateChange BLOCK* processReport: from storage DS-008ac67c-e28d-479f-9f37-ebe4d7657e79 node DatanodeRegistration(127.0.0.1:41286, datanodeUuid=567a9e45-ef67-4d16-bc60-5d66dab05f16, infoPort=48705, infoSecurePort=0, ipcPort=59273, storageInfo=lv=-56;cid=testClusterID;nsid=949490773;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs
[junit4] 2> 1521151 INFO (IPC Server handler 5 on 40454) [ ] BlockStateChange BLOCK* processReport: from storage DS-e0fdfa24-b678-453a-bcdb-465e77cdcd29 node DatanodeRegistration(127.0.0.1:41286, datanodeUuid=567a9e45-ef67-4d16-bc60-5d66dab05f16, infoPort=48705, infoSecurePort=0, ipcPort=59273, storageInfo=lv=-56;cid=testClusterID;nsid=949490773;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 1521283 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1521284 INFO (Thread-57582) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1521284 INFO (Thread-57582) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1521285 ERROR (Thread-57582) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1521384 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.ZkTestServer start zk server on port:39926
[junit4] 2> 1521538 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1521540 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 1521542 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1521544 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1521545 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1521546 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1521548 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1521549 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1521550 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1521551 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1521553 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1521555 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
[junit4] 2> 1522002 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1522034 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1442a9a{/m_/r,null,AVAILABLE}
[junit4] 2> 1522035 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4d7e62f2{HTTP/1.1,[http/1.1]}{127.0.0.1:57502}
[junit4] 2> 1522035 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.e.j.s.Server Started @1526566ms
[junit4] 2> 1522035 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:40454/hdfs__localhost_40454__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.0_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001_tempDir-002_control_data, replicaType=NRT, hostContext=/m_/r, hostPort=57502, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001/control-001/cores}
[junit4] 2> 1522035 ERROR (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1522035 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1522036 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1522036 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1522036 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-09T23:04:57.960Z
[junit4] 2> 1522072 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1522072 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001/control-001/solr.xml
[junit4] 2> 1522078 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@461a898c, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1522125 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39926/solr
[junit4] 2> 1522314 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1522314 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:57502_m_%2Fr
[junit4] 2> 1522315 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.Overseer Overseer (id=98456036783489028-127.0.0.1:57502_m_%2Fr-n_0000000000) starting
[junit4] 2> 1522359 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57502_m_%2Fr
[junit4] 2> 1522365 INFO (zkCallback-925-thread-1-processing-n:127.0.0.1:57502_m_%2Fr) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1522543 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@461a898c
[junit4] 2> 1522549 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@461a898c
[junit4] 2> 1522549 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@461a898c
[junit4] 2> 1522551 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001/control-001/cores
[junit4] 2> 1522604 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1522605 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39926/solr ready
[junit4] 2> 1522606 INFO (qtp1270481482-62781) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:57502_m_%252Fr&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1522613 INFO (OverseerThreadFactory-1953-thread-1-processing-n:127.0.0.1:57502_m_%2Fr) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 1522781 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1522781 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1522904 INFO (zkCallback-925-thread-1-processing-n:127.0.0.1:57502_m_%2Fr) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1523816 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1523892 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 1524288 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1524360 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 1524361 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@461a898c
[junit4] 2> 1524361 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:40454/solr_hdfs_home
[junit4] 2> 1524361 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 1524361 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1524361 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://localhost:40454/solr_hdfs_home/control_collection/core_node1/data/]
[junit4] 2> 1524363 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40454/solr_hdfs_home/control_collection/core_node1/data/snapshot_metadata
[junit4] 2> 1524380 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 1524380 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 1524381 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 1524395 WARN (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 1530177 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 1530178 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40454/solr_hdfs_home/control_collection/core_node1/data
[junit4] 2> 1530202 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40454/solr_hdfs_home/control_collection/core_node1/data/index
[junit4] 2> 1530212 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 1530212 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 1530213 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 1530224 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 1530225 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=24.673828125, floorSegmentMB=1.5146484375, forceMergeDeletesPctAllowed=3.905641746415954, segmentsPerTier=3.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7781730775016806
[junit4] 2> 1530273 INFO (IPC Server handler 3 on 40454) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:53909 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-e0fdfa24-b678-453a-bcdb-465e77cdcd29:NORMAL:127.0.0.1:41286|RBW], ReplicaUC[[DISK]DS-9b8d8cc1-1c67-4296-93ce-c70c85c0e635:NORMAL:127.0.0.1:53909|RBW]]} size 0
[junit4] 2> 1530277 INFO (IPC Server handler 4 on 40454) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41286 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-9b8d8cc1-1c67-4296-93ce-c70c85c0e635:NORMAL:127.0.0.1:53909|RBW], ReplicaUC[[DISK]DS-008ac67c-e28d-479f-9f37-ebe4d7657e79:NORMAL:127.0.0.1:41286|FINALIZED]]} size 0
[junit4] 2> 1530290 WARN (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1530391 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 1530391 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1530391 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 1530402 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1530402 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1530438 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=26, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7839834230981928]
[junit4] 2> 1530493 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@341ced72[control_collection_shard1_replica_n1] main]
[junit4] 2> 1530495 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1530496 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1530496 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1530497 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1575296598195306496
[junit4] 2> 1530501 INFO (searcherExecutor-1956-thread-1-processing-n:127.0.0.1:57502_m_%2Fr x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@341ced72[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1530506 INFO (zkCallback-925-thread-1-processing-n:127.0.0.1:57502_m_%2Fr) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1530508 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1530508 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1530508 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:57502/m_/r/control_collection_shard1_replica_n1/
[junit4] 2> 1530508 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1530508 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:57502/m_/r/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 1530509 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1530517 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:57502/m_/r/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 1530620 INFO (zkCallback-925-thread-1-processing-n:127.0.0.1:57502_m_%2Fr) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1530668 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1530671 INFO (qtp1270481482-62780) [n:127.0.0.1:57502_m_%2Fr c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=7890
[junit4] 2> 1530681 INFO (qtp1270481482-62781) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1530775 INFO (zkCallback-925-thread-1-processing-n:127.0.0.1:57502_m_%2Fr) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1531682 INFO (qtp1270481482-62781) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:57502_m_%252Fr&wt=javabin&version=2} status=0 QTime=9075
[junit4] 2> 1531689 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1531690 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39926/solr ready
[junit4] 2> 1531690 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1531691 INFO (qtp1270481482-62783) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1531695 INFO (OverseerCollectionConfigSetProcessor-98456036783489028-127.0.0.1:57502_m_%2Fr-n_0000000000) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1531706 INFO (OverseerThreadFactory-1953-thread-2-processing-n:127.0.0.1:57502_m_%2Fr) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 1531707 WARN (OverseerThreadFactory-1953-thread-2-processing-n:127.0.0.1:57502_m_%2Fr) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 1531924 INFO (qtp1270481482-62783) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1531926 INFO (qtp1270481482-62783) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=235
[junit4] 2> 1532191 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001/shard-1-001 of type TLOG
[junit4] 2> 1532191 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1532193 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@13c3fe58{/m_/r,null,AVAILABLE}
[junit4] 2> 1532193 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6880ff6d{HTTP/1.1,[http/1.1]}{127.0.0.1:59297}
[junit4] 2> 1532193 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.e.j.s.Server Started @1536725ms
[junit4] 2> 1532193 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:40454/hdfs__localhost_40454__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.0_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001_tempDir-002_jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/m_/r, hostPort=59297, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001/shard-1-001/cores}
[junit4] 2> 1532194 ERROR (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1532194 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1532194 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1532194 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1532194 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-09T23:05:08.118Z
[junit4] 2> 1532205 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1532205 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001/shard-1-001/solr.xml
[junit4] 2> 1532224 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@461a898c, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1532227 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39926/solr
[junit4] 2> 1532249 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1532252 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1532254 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59297_m_%2Fr
[junit4] 2> 1532256 INFO (zkCallback-932-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1532262 INFO (zkCallback-925-thread-1-processing-n:127.0.0.1:57502_m_%2Fr) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1532274 INFO (zkCallback-937-thread-1-processing-n:127.0.0.1:59297_m_%2Fr) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1532407 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@461a898c
[junit4] 2> 1532429 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@461a898c
[junit4] 2> 1532430 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@461a898c
[junit4] 2> 1532432 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001/shard-1-001/cores
[junit4] 2> 1532462 INFO (qtp1399615956-62835) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:59297_m_%252Fr&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1532469 INFO (OverseerCollectionConfigSetProcessor-98456036783489028-127.0.0.1:57502_m_%2Fr-n_0000000000) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1532473 INFO (OverseerThreadFactory-1953-thread-3-processing-n:127.0.0.1:57502_m_%2Fr) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:59297_m_%2Fr for creating new replica
[junit4] 2> 1532477 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 1532478 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1532591 INFO (zkCallback-937-thread-1-processing-n:127.0.0.1:59297_m_%2Fr) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1533504 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1533520 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.s.IndexSchema [collection1_shard2_replica_t0] Schema name=test
[junit4] 2> 1533741 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1533779 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t0' using configuration from collection collection1, trusted=true
[junit4] 2> 1533779 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t0' (registry 'solr.core.collection1.shard2.replica_t0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@461a898c
[junit4] 2> 1533780 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:40454/solr_hdfs_home
[junit4] 2> 1533780 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 1533780 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1533780 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SolrCore [[collection1_shard2_replica_t0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001/shard-1-001/cores/collection1_shard2_replica_t0], dataDir=[hdfs://localhost:40454/solr_hdfs_home/collection1/core_node1/data/]
[junit4] 2> 1533781 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40454/solr_hdfs_home/collection1/core_node1/data/snapshot_metadata
[junit4] 2> 1533791 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 1533791 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 1533791 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 1533797 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 1533798 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40454/solr_hdfs_home/collection1/core_node1/data
[junit4] 2> 1533819 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40454/solr_hdfs_home/collection1/core_node1/data/index
[junit4] 2> 1533838 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 1533838 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 1533838 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 1533850 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 1533851 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=24.673828125, floorSegmentMB=1.5146484375, forceMergeDeletesPctAllowed=3.905641746415954, segmentsPerTier=3.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7781730775016806
[junit4] 2> 1533884 INFO (IPC Server handler 3 on 40454) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41286 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-9f61f565-dda2-49d1-8724-7c4f4e0c12c5:NORMAL:127.0.0.1:53909|RBW], ReplicaUC[[DISK]DS-e0fdfa24-b678-453a-bcdb-465e77cdcd29:NORMAL:127.0.0.1:41286|FINALIZED]]} size 0
[junit4] 2> 1533884 INFO (IPC Server handler 9 on 40454) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:53909 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-9f61f565-dda2-49d1-8724-7c4f4e0c12c5:NORMAL:127.0.0.1:53909|RBW], ReplicaUC[[DISK]DS-e0fdfa24-b678-453a-bcdb-465e77cdcd29:NORMAL:127.0.0.1:41286|FINALIZED]]} size 0
[junit4] 2> 1533893 WARN (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1533983 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 1533983 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1533983 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 1533996 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1533996 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1533998 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=26, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7839834230981928]
[junit4] 2> 1534003 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.s.SolrIndexSearcher Opening [Searcher@66cfc45f[collection1_shard2_replica_t0] main]
[junit4] 2> 1534005 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1534005 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1534006 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1534008 INFO (searcherExecutor-1967-thread-1-processing-n:127.0.0.1:59297_m_%2Fr x:collection1_shard2_replica_t0 s:shard2 c:collection1) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SolrCore [collection1_shard2_replica_t0] Registered new searcher Searcher@66cfc45f[collection1_shard2_replica_t0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1534008 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1575296601876856832
[junit4] 2> 1534017 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1534017 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1534017 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:59297/m_/r/collection1_shard2_replica_t0/
[junit4] 2> 1534017 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1534018 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SyncStrategy http://127.0.0.1:59297/m_/r/collection1_shard2_replica_t0/ has no replicas
[junit4] 2> 1534018 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1534018 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ZkController collection1_shard2_replica_t0 stopping background replication from leader
[junit4] 2> 1534028 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:59297/m_/r/collection1_shard2_replica_t0/ shard2
[junit4] 2> 1534132 INFO (zkCallback-937-thread-1-processing-n:127.0.0.1:59297_m_%2Fr) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1534184 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1534190 INFO (qtp1399615956-62833) [n:127.0.0.1:59297_m_%2Fr c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1713
[junit4] 2> 1534197 INFO (qtp1399615956-62835) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:59297_m_%252Fr&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=1734
[junit4] 2> 1534299 INFO (zkCallback-937-thread-1-processing-n:127.0.0.1:59297_m_%2Fr) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1534373 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001/shard-2-001 of type TLOG
[junit4] 2> 1534374 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1534375 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5b108f57{/m_/r,null,AVAILABLE}
[junit4] 2> 1534376 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@67d2c775{HTTP/1.1,[http/1.1]}{127.0.0.1:33224}
[junit4] 2> 1534376 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.e.j.s.Server Started @1538907ms
[junit4] 2> 1534376 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:40454/hdfs__localhost_40454__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.0_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001_tempDir-002_jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/m_/r, hostPort=33224, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001/shard-2-001/cores}
[junit4] 2> 1534377 ERROR (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1534378 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1534378 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1534378 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1534378 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-09T23:05:10.302Z
[junit4] 2> 1534390 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1534390 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001/shard-2-001/solr.xml
[junit4] 2> 1534396 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@461a898c, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1534400 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39926/solr
[junit4] 2> 1534434 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:33224_m_%2Fr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1534440 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:33224_m_%2Fr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1534442 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:33224_m_%2Fr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33224_m_%2Fr
[junit4] 2> 1534444 INFO (zkCallback-932-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1534445 INFO (zkCallback-937-thread-1-processing-n:127.0.0.1:59297_m_%2Fr) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1534445 INFO (zkCallback-925-thread-1-processing-n:127.0.0.1:57502_m_%2Fr) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1534467 INFO (zkCallback-943-thread-1-processing-n:127.0.0.1:33224_m_%2Fr) [n:127.0.0.1:33224_m_%2Fr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1534475 INFO (OverseerCollectionConfigSetProcessor-98456036783489028-127.0.0.1:57502_m_%2Fr-n_0000000000) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1534548 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:33224_m_%2Fr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@461a898c
[junit4] 2> 1534617 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:33224_m_%2Fr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@461a898c
[junit4] 2> 1534617 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:33224_m_%2Fr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@461a898c
[junit4] 2> 1534620 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[177FA75A9BC052C6]) [n:127.0.0.1:33224_m_%2Fr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001/shard-2-001/cores
[junit4] 2> 1534656 INFO (qtp1399615956-62836) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:33224_m_%252Fr&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1534661 INFO (OverseerThreadFactory-1953-thread-4-processing-n:127.0.0.1:57502_m_%2Fr) [n:127.0.0.1:57502_m_%2Fr ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:33224_m_%2Fr for creating new replica
[junit4] 2> 1534665 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t0&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 1534665 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1534770 INFO (zkCallback-943-thread-1-processing-n:127.0.0.1:33224_m_%2Fr) [n:127.0.0.1:33224_m_%2Fr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1534770 INFO (zkCallback-937-thread-1-processing-n:127.0.0.1:59297_m_%2Fr) [n:127.0.0.1:59297_m_%2Fr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1535682 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1535697 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.s.IndexSchema [collection1_shard1_replica_t0] Schema name=test
[junit4] 2> 1535912 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1535946 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t0' using configuration from collection collection1, trusted=true
[junit4] 2> 1535947 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t0' (registry 'solr.core.collection1.shard1.replica_t0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@461a898c
[junit4] 2> 1535947 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:40454/solr_hdfs_home
[junit4] 2> 1535947 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 1535947 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1535947 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SolrCore [[collection1_shard1_replica_t0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_177FA75A9BC052C6-001/shard-2-001/cores/collection1_shard1_replica_t0], dataDir=[hdfs://localhost:40454/solr_hdfs_home/collection1/core_node2/data/]
[junit4] 2> 1535949 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40454/solr_hdfs_home/collection1/core_node2/data/snapshot_metadata
[junit4] 2> 1535957 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 1535958 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 1535958 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 1535963 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 1535964 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40454/solr_hdfs_home/collection1/core_node2/data
[junit4] 2> 1535991 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:40454/solr_hdfs_home/collection1/core_node2/data/index
[junit4] 2> 1535999 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 1535999 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 1535999 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 1536061 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 1536063 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=24.673828125, floorSegmentMB=1.5146484375, forceMergeDeletesPctAllowed=3.905641746415954, segmentsPerTier=3.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7781730775016806
[junit4] 2> 1536163 INFO (IPC Server handler 1 on 40454) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:53909 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-008ac67c-e28d-479f-9f37-ebe4d7657e79:NORMAL:127.0.0.1:41286|RBW], ReplicaUC[[DISK]DS-9b8d8cc1-1c67-4296-93ce-c70c85c0e635:NORMAL:127.0.0.1:53909|FINALIZED]]} size 0
[junit4] 2> 1536164 INFO (IPC Server handler 2 on 40454) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41286 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-008ac67c-e28d-479f-9f37-ebe4d7657e79:NORMAL:127.0.0.1:41286|RBW], ReplicaUC[[DISK]DS-9b8d8cc1-1c67-4296-93ce-c70c85c0e635:NORMAL:127.0.0.1:53909|FINALIZED]]} size 0
[junit4] 2> 1536190 WARN (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1536364 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 1536364 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1536364 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 1536381 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1536381 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1536392 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=26, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7839834230981928]
[junit4] 2> 1536435 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.s.SolrIndexSearcher Opening [Searcher@3c8b4361[collection1_shard1_replica_t0] main]
[junit4] 2> 1536437 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1536441 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1536442 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1536443 INFO (searcherExecutor-1978-thread-1-processing-n:127.0.0.1:33224_m_%2Fr x:collection1_shard1_replica_t0 s:shard1 c:collection1) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SolrCore [collection1_shard1_replica_t0] Registered new searcher Searcher@3c8b4361[collection1_shard1_replica_t0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1536444 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1575296604431187968
[junit4] 2> 1536456 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1536456 INFO (qtp233554580-62872) [n:127.0.0.1:33224_m_%2Fr c:collection1 s:shard1 x:collection1_shard1_replica_t0] o
[...truncated too long message...]
equestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/get,name=requestTimes, solr:dom1=node,category=QUERY,scope=/admin/metrics/collector,name=clientErrors, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=threadPool.updateExecutor.completed, solr:dom1=core,dom2=collection1,category=QUERY,scope=/get,name=clientErrors, solr:dom1=node,category=ADMIN,scope=/admin/authorization,name=totalTime, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n2,category=QUERY,scope=/dump,name=handlerStart, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json,name=handlerStart, solr:dom1=node,category=QUERY,scope=httpShardHandler,name=threadPool.httpShardExecutor.submitted, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/system,name=errors, solr:dom1=node,category=QUERY,scope=httpShardHandler,name=threadPool.httpShardExecutor.running, solr:dom1=core,dom2=collection1,category=QUERY,scope=/debug/dump,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n2,category=UPDATE,scope=/update/json,name=handlerStart, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Eden-Space,name=usage, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update,name=requests, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Eden-Space,name=used, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/properties,name=requestTimes, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=indexVersion, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n2,category=QUERY,scope=/select,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mlt,name=timeouts, solr:dom1=node,category=ADMIN,scope=/admin/configs,name=clientErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n2,category=REPLICATION,scope=/replication,name=generation, solr:dom1=core,dom2=collection1,category=QUERY,scope=/stream,name=handlerStart, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n2,category=QUERY,scope=/graph,name=totalTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/mbeans,name=totalTime, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json,name=requests, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n2,category=ADMIN,scope=/admin/file,name=handlerStart, solr:dom1=jvm,name0=memory,name1=total,name=committed, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n2,category=HIGHLIGHTER,scope=SolrFragmentsBuilder,name=default.requests, solr:dom1=node,category=QUERY,scope=/admin/metrics/collector,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/debug/dump,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n2,category=UPDATE,scope=/update/json,name=timeouts, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n2,category=UPDATE,scope=updateHandler,name=splits, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/system,name=serverErrors, solr:dom1=jvm,name0=threads,name=deadlocks, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n2,category=ADMIN,scope=/admin/ping,name=errors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n2,category=UPDATE,scope=update,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak_Direct,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH_Direct,name=serverErrors, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n2,category=REPLICATION,scope=/replication,name=requests, solr:dom1=jvm,name0=memory,name1=non-heap,name=usage, solr:dom1=jvm,name0=classes,name=loaded, solr:dom1=core,dom2=collection1,dom3=shard1,dom4=replica_n2,category=REPLICATION,scope=/replication,name=replicationEnabled, java.lang:type=Memory, solr:dom1=jvm,name0=memory,name1=total,name=init, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/logging,name=errors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/luke,name=requestTimes, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=leasedConnections, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH,name=timeouts, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrFragListBuilder,name=simple.requests]
[junit4] 2> 2908524 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[177FA75A9BC052C6]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[1 (1575298042937999360)]} 0 1
[junit4] 2> 2908527 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[177FA75A9BC052C6]) [ ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1575298042942193664,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2908527 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[177FA75A9BC052C6]) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@11f08a3d commitCommandVersion:1575298042942193664
[junit4] 2> 2908530 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[177FA75A9BC052C6]) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@3b95204b[collection1] main]
[junit4] 2> 2908531 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[177FA75A9BC052C6]) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2908534 INFO (searcherExecutor-7174-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3b95204b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):C1)))}
[junit4] 2> 2908534 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[177FA75A9BC052C6]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{commit=} 0 7
[junit4] 2> 2908534 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[177FA75A9BC052C6]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testJmxUpdate
[junit4] 2> 2908535 INFO (SUITE-TestJmxIntegration-seed#[177FA75A9BC052C6]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> 2908535 INFO (SUITE-TestJmxIntegration-seed#[177FA75A9BC052C6]-worker) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=935938321
[junit4] 2> 2908535 INFO (SUITE-TestJmxIntegration-seed#[177FA75A9BC052C6]-worker) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 2908537 INFO (SUITE-TestJmxIntegration-seed#[177FA75A9BC052C6]-worker) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 2908554 INFO (SUITE-TestJmxIntegration-seed#[177FA75A9BC052C6]-worker) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 2908555 INFO (SUITE-TestJmxIntegration-seed#[177FA75A9BC052C6]-worker) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2908562 INFO (coreCloseExecutor-7179-thread-1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@70df768e
[junit4] 2> 2908563 INFO (coreCloseExecutor-7179-thread-1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1893693070
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J2/temp/solr.core.TestJmxIntegration_177FA75A9BC052C6-001
[junit4] 2> Aug 09, 2017 11:28:04 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=false): {}, locale=th, timezone=America/Indianapolis
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=105714552,total=523239424
[junit4] 2> NOTE: All tests run in this JVM: [TestManagedResource, TestSimpleQParserPlugin, SpellPossibilityIteratorTest, UtilsToolTest, TestDistributedMissingSort, HttpSolrCallGetCoreTest, CollectionStateFormat2Test, CircularListTest, TestHdfsBackupRestoreCore, TestMinMaxOnMultiValuedField, BadCopyFieldTest, CollectionsAPISolrJTest, TestConfigSetImmutable, DeleteInactiveReplicaTest, TestLegacyField, TestZkChroot, AliasIntegrationTest, ConnectionManagerTest, TestLFUCache, RulesTest, PreAnalyzedUpdateProcessorTest, TestOnReconnectListenerSupport, TestRebalanceLeaders, HdfsRestartWhileUpdatingTest, LeaderFailoverAfterPartitionTest, SearchHandlerTest, SignatureUpdateProcessorFactoryTest, TestSurroundQueryParser, HdfsChaosMonkeyNothingIsSafeTest, TestAnalyzedSuggestions, TestSolrCoreParser, DocExpirationUpdateProcessorFactoryTest, TestReversedWildcardFilterFactory, PeerSyncReplicationTest, TestCustomSort, TestExportWriter, RollingRestartTest, ClassificationUpdateProcessorIntegrationTest, DistribCursorPagingTest, CopyFieldTest, BlockCacheTest, TestSolrConfigHandlerCloud, TestFileDictionaryLookup, CleanupOldIndexTest, FullHLLTest, CoreAdminRequestStatusTest, PKIAuthenticationIntegrationTest, UniqFieldsUpdateProcessorFactoryTest, TestTolerantSearch, MultiThreadedOCPTest, TestLegacyFieldCache, TestStressUserVersions, MultiTermTest, TermVectorComponentDistributedTest, SpatialHeatmapFacetsTest, CdcrUpdateLogTest, PingRequestHandlerTest, MetricsHandlerTest, TestSolrCloudWithKerberosAlt, ReturnFieldsTest, TestJsonRequest, TestSimpleTextCodec, TestSolrDeletionPolicy2, SubstringBytesRefFilterTest, StressHdfsTest, TestDistributedStatsComponentCardinality, SimpleCollectionCreateDeleteTest, TestDistribDocBasedVersion, StatsComponentTest, AutoCommitTest, TestMacroExpander, TestHdfsCloudBackupRestore, TestJmxIntegration]
[junit4] Completed [312/728 (3!)] on J2 in 0.97s, 3 tests, 1 error, 1 skipped <<< FAILURES!
[...truncated 44719 lines...]