You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/10/29 03:28:21 UTC
[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 361 - Still
Unstable
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/361/
4 tests failed.
FAILED: org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest.test
Error Message:
There are still nodes recoverying - waited for 320 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
at __randomizedtesting.SeedInfo.seed([F900155FC4E8D440:71542A856A14B9B8]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:920)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1477)
at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:145)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1010)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:985)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
Error Message:
7 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest: 1) Thread[id=73979, name=searcherExecutor-9510-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=73855, name=searcherExecutor-9468-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=74059, name=searcherExecutor-9530-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=73935, name=searcherExecutor-9496-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=74094, name=searcherExecutor-9544-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=73898, name=searcherExecutor-9482-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=74129, name=searcherExecutor-9558-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 7 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest:
1) Thread[id=73979, name=searcherExecutor-9510-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=73855, name=searcherExecutor-9468-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=74059, name=searcherExecutor-9530-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=73935, name=searcherExecutor-9496-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=74094, name=searcherExecutor-9544-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=73898, name=searcherExecutor-9482-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=74129, name=searcherExecutor-9558-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([F900155FC4E8D440]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=73979, name=searcherExecutor-9510-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=73855, name=searcherExecutor-9468-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=74059, name=searcherExecutor-9530-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=73935, name=searcherExecutor-9496-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=74094, name=searcherExecutor-9544-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=73898, name=searcherExecutor-9482-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=74129, name=searcherExecutor-9558-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=73979, name=searcherExecutor-9510-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=73855, name=searcherExecutor-9468-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=74059, name=searcherExecutor-9530-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=73935, name=searcherExecutor-9496-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=74094, name=searcherExecutor-9544-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=73898, name=searcherExecutor-9482-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=74129, name=searcherExecutor-9558-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([F900155FC4E8D440]:0)
FAILED: org.apache.solr.client.solrj.impl.CloudSolrClientTest.testRouting
Error Message:
Error from server at https://127.0.0.1:44030/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> <title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title> </head> <body><h2>HTTP ERROR 404</h2> <p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason: <pre> Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/> </body> </html>
Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:44030/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title>
</head>
<body><h2>HTTP ERROR 404</h2>
<p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason:
<pre> Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/>
</body>
</html>
at __randomizedtesting.SeedInfo.seed([1D4E01942286A0E8:DFF93DFC21C65090]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:551)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1016)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.impl.CloudSolrClientTest.testRouting(CloudSolrClientTest.java:238)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
at org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:110)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:44030/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title>
</head>
<body><h2>HTTP ERROR 404</h2>
<p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason:
<pre> Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/>
</body>
</html>
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:528)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Build Log:
[...truncated 15600 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
[junit4] 2> 3587569 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/init-core-data-001
[junit4] 2> 3587570 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 3587573 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 3587573 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 3587612 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 3587618 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3587620 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3587655 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_36929_hdfs____gftciv/webapp
[junit4] 2> 3588066 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36929
[junit4] 2> 3588154 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3588155 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3588177 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_33140_datanode____ry3w50/webapp
[junit4] 2> 3588593 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33140
[junit4] 2> 3588651 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3588652 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3588683 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_40985_datanode____4bf7xd/webapp
[junit4] 2> 3588728 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:37804) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 3588740 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x6cf25facef291: from storage DS-ebed7a1a-1f24-4caf-a0c9-5368e57332ce node DatanodeRegistration(127.0.0.1:41525, datanodeUuid=d95058ce-0cf7-481e-a2de-76934e48249e, infoPort=42099, infoSecurePort=0, ipcPort=39795, storageInfo=lv=-56;cid=testClusterID;nsid=1850678792;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 3588741 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x6cf25facef291: from storage DS-93aab6fb-2a44-4ee3-b7d0-4874459362a8 node DatanodeRegistration(127.0.0.1:41525, datanodeUuid=d95058ce-0cf7-481e-a2de-76934e48249e, infoPort=42099, infoSecurePort=0, ipcPort=39795, storageInfo=lv=-56;cid=testClusterID;nsid=1850678792;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 3589130 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40985
[junit4] 2> 3589258 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:37804) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 3589271 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x6cf261a745818: from storage DS-d042ba87-bfe4-4b03-b952-06e6806a80bb node DatanodeRegistration(127.0.0.1:40487, datanodeUuid=2966e4fd-6e7b-460a-8f46-1c3cd2d8bd22, infoPort=41526, infoSecurePort=0, ipcPort=33525, storageInfo=lv=-56;cid=testClusterID;nsid=1850678792;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 3589272 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x6cf261a745818: from storage DS-e07e0b27-5421-4d40-89a3-d8e8666ab47c node DatanodeRegistration(127.0.0.1:40487, datanodeUuid=2966e4fd-6e7b-460a-8f46-1c3cd2d8bd22, infoPort=41526, infoSecurePort=0, ipcPort=33525, storageInfo=lv=-56;cid=testClusterID;nsid=1850678792;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 3589457 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3589457 INFO (Thread-55028) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3589457 INFO (Thread-55028) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3589461 ERROR (Thread-55028) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3589557 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.ZkTestServer start zk server on port:35789
[junit4] 2> 3589562 INFO (zkConnectionManagerCallback-6720-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3589566 INFO (zkConnectionManagerCallback-6722-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3589573 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 3589575 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 3589577 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 3589578 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 3589580 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 3589582 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 3589583 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 3589585 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 3589586 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 3589588 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 3589589 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 3589591 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 3589765 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 3589769 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3589769 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3589769 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 3589769 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2c07bd5a{/,null,AVAILABLE}
[junit4] 2> 3589770 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2cbbf67d{HTTP/1.1,[http/1.1]}{127.0.0.1:44864}
[junit4] 2> 3589770 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.Server Started @3589829ms
[junit4] 2> 3589770 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:37804/hdfs__localhost_37804__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001_tempDir-002_control_data, replicaType=NRT, hostContext=/, hostPort=44864, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/control-001/cores}
[junit4] 2> 3589773 ERROR (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3589773 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 3589773 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 3589773 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3589773 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3589773 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-29T02:57:07.150Z
[junit4] 2> 3589775 INFO (zkConnectionManagerCallback-6724-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3589776 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3589777 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/control-001/solr.xml
[junit4] 2> 3589781 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3589781 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3589783 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3589963 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35789/solr
[junit4] 2> 3589964 INFO (zkConnectionManagerCallback-6728-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3589969 INFO (zkConnectionManagerCallback-6730-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3590082 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:44864_
[junit4] 2> 3590084 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.c.Overseer Overseer (id=72183201161740292-127.0.0.1:44864_-n_0000000000) starting
[junit4] 2> 3590093 INFO (zkConnectionManagerCallback-6737-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3590095 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35789/solr ready
[junit4] 2> 3590096 INFO (OverseerStateUpdate-72183201161740292-127.0.0.1:44864_-n_0000000000) [n:127.0.0.1:44864_ ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:44864_
[junit4] 2> 3590104 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44864_
[junit4] 2> 3590112 INFO (zkCallback-6736-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3590112 INFO (zkCallback-6729-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3590134 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 3590170 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150
[junit4] 2> 3590188 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150
[junit4] 2> 3590189 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150
[junit4] 2> 3590191 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/control-001/cores
[junit4] 2> 3590221 INFO (zkConnectionManagerCallback-6743-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3590223 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3590225 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35789/solr ready
[junit4] 2> 3590226 INFO (qtp379481793-73815) [n:127.0.0.1:44864_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:44864_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3590234 INFO (OverseerThreadFactory-9463-thread-1-processing-n:127.0.0.1:44864_) [n:127.0.0.1:44864_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 3590343 INFO (qtp379481793-73810) [n:127.0.0.1:44864_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3590349 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3590349 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3591375 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 3591392 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 3591550 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 3591573 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 3591574 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150
[junit4] 2> 3591574 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 3591583 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=1.73828125, floorSegmentMB=1.2861328125, forceMergeDeletesPctAllowed=17.251147289491627, segmentsPerTier=27.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19017649322459562, deletesPctAllowed=36.714809707763436
[junit4] 2> 3591595 WARN (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3591687 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3591687 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3591693 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 30000ms;
[junit4] 2> 3591693 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
[junit4] 2> 3591695 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=35, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=0.7802734375, floorSegmentMB=0.8623046875, forceMergeDeletesPctAllowed=28.347754521320784, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.48483855280643395, deletesPctAllowed=25.385944100098243
[junit4] 2> 3591696 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1cd47ed[control_collection_shard1_replica_n1] main]
[junit4] 2> 3591702 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3591703 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3591704 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3591706 INFO (searcherExecutor-9468-thread-1-processing-n:127.0.0.1:44864_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@1cd47ed[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3591706 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615626847212535808
[junit4] 2> 3591716 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 3591722 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3591722 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3591722 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:44864/control_collection_shard1_replica_n1/
[junit4] 2> 3591722 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3591723 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:44864/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 3591723 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3591728 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:44864/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 3591832 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3591834 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1485
[junit4] 2> 3591840 INFO (qtp379481793-73815) [n:127.0.0.1:44864_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3591935 INFO (zkCallback-6729-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3592239 INFO (OverseerCollectionConfigSetProcessor-72183201161740292-127.0.0.1:44864_-n_0000000000) [n:127.0.0.1:44864_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3592840 INFO (qtp379481793-73815) [n:127.0.0.1:44864_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:44864_&wt=javabin&version=2} status=0 QTime=2613
[junit4] 2> 3592845 INFO (zkConnectionManagerCallback-6748-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3592846 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3592848 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35789/solr ready
[junit4] 2> 3592848 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 3592849 INFO (qtp379481793-73814) [n:127.0.0.1:44864_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3592852 INFO (OverseerThreadFactory-9463-thread-2-processing-n:127.0.0.1:44864_) [n:127.0.0.1:44864_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 3593055 WARN (OverseerThreadFactory-9463-thread-2-processing-n:127.0.0.1:44864_) [n:127.0.0.1:44864_ ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 3593058 INFO (qtp379481793-73814) [n:127.0.0.1:44864_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3593058 INFO (qtp379481793-73814) [n:127.0.0.1:44864_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=209
[junit4] 2> 3593237 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/shard-1-001 of type NRT
[junit4] 2> 3593238 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 3593239 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3593239 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3593239 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 3593240 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@84f52b{/,null,AVAILABLE}
[junit4] 2> 3593240 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@23395dc9{HTTP/1.1,[http/1.1]}{127.0.0.1:36388}
[junit4] 2> 3593242 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.Server Started @3593301ms
[junit4] 2> 3593242 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:37804/hdfs__localhost_37804__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001_tempDir-002_jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=36388, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/shard-1-001/cores}
[junit4] 2> 3593244 ERROR (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3593244 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 3593244 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 3593244 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3593244 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3593244 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-29T02:57:10.621Z
[junit4] 2> 3593246 INFO (zkConnectionManagerCallback-6750-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3593247 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3593247 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/shard-1-001/solr.xml
[junit4] 2> 3593252 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3593252 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3593299 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3593457 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35789/solr
[junit4] 2> 3593458 INFO (zkConnectionManagerCallback-6754-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3593461 INFO (zkConnectionManagerCallback-6756-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3593469 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3593474 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3593474 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36388_
[junit4] 2> 3593476 INFO (zkCallback-6736-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3593483 INFO (zkCallback-6729-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3593483 INFO (zkCallback-6747-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3593488 INFO (zkCallback-6755-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3593506 INFO (zkConnectionManagerCallback-6763-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3593507 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3593508 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35789/solr ready
[junit4] 2> 3593509 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 3593539 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150
[junit4] 2> 3593558 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150
[junit4] 2> 3593559 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150
[junit4] 2> 3593561 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/shard-1-001/cores
[junit4] 2> 3593605 INFO (qtp36025874-73870) [n:127.0.0.1:36388_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:36388_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3593608 INFO (OverseerCollectionConfigSetProcessor-72183201161740292-127.0.0.1:44864_-n_0000000000) [n:127.0.0.1:44864_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3593611 INFO (qtp36025874-73865) [n:127.0.0.1:36388_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3593616 INFO (qtp379481793-73817) [n:127.0.0.1:44864_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=1
[junit4] 2> 3593618 INFO (qtp379481793-73817) [n:127.0.0.1:44864_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
[junit4] 2> 3593618 INFO (OverseerThreadFactory-9463-thread-3-processing-n:127.0.0.1:44864_) [n:127.0.0.1:44864_ c:collection1 s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:36388_ for creating new replica of shard shard1 for collection collection1
[junit4] 2> 3593625 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ x:collection1_shard1_replica_n21] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3594649 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 3594672 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema [collection1_shard1_replica_n21] Schema name=test
[junit4] 2> 3594791 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 3594810 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n21' using configuration from collection collection1, trusted=true
[junit4] 2> 3594811 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n21' (registry 'solr.core.collection1.shard1.replica_n21') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150
[junit4] 2> 3594811 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [[collection1_shard1_replica_n21] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/shard-1-001/cores/collection1_shard1_replica_n21], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/shard-1-001/cores/collection1_shard1_replica_n21/data/]
[junit4] 2> 3594816 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=1.73828125, floorSegmentMB=1.2861328125, forceMergeDeletesPctAllowed=17.251147289491627, segmentsPerTier=27.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19017649322459562, deletesPctAllowed=36.714809707763436
[junit4] 2> 3594824 WARN (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3594895 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3594896 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3594897 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 30000ms;
[junit4] 2> 3594897 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
[junit4] 2> 3594900 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=35, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=0.7802734375, floorSegmentMB=0.8623046875, forceMergeDeletesPctAllowed=28.347754521320784, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.48483855280643395, deletesPctAllowed=25.385944100098243
[junit4] 2> 3594900 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.SolrIndexSearcher Opening [Searcher@44694623[collection1_shard1_replica_n21] main]
[junit4] 2> 3594902 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3594902 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3594903 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3594903 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615626850564833280
[junit4] 2> 3594907 INFO (searcherExecutor-9482-thread-1-processing-n:127.0.0.1:36388_ x:collection1_shard1_replica_n21 c:collection1 s:shard1) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [collection1_shard1_replica_n21] Registered new searcher Searcher@44694623[collection1_shard1_replica_n21] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3594911 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node22=0}, version=0}
[junit4] 2> 3594915 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3594915 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3594915 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:36388/collection1_shard1_replica_n21/
[junit4] 2> 3594915 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3594915 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy http://127.0.0.1:36388/collection1_shard1_replica_n21/ has no replicas
[junit4] 2> 3594915 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3594919 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:36388/collection1_shard1_replica_n21/ shard1
[junit4] 2> 3595071 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3595074 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1449
[junit4] 2> 3595077 INFO (qtp36025874-73870) [n:127.0.0.1:36388_ c:collection1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:36388_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1471
[junit4] 2> 3595286 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/shard-2-001 of type NRT
[junit4] 2> 3595288 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 3595289 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3595289 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3595289 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 3595290 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@44694501{/,null,AVAILABLE}
[junit4] 2> 3595290 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6177fc40{HTTP/1.1,[http/1.1]}{127.0.0.1:46131}
[junit4] 2> 3595290 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.Server Started @3595349ms
[junit4] 2> 3595290 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:37804/hdfs__localhost_37804__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001_tempDir-002_jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=46131, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/shard-2-001/cores}
[junit4] 2> 3595291 ERROR (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3595291 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 3595291 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 3595291 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3595291 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3595294 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-29T02:57:12.671Z
[junit4] 2> 3595297 INFO (zkConnectionManagerCallback-6766-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3595298 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3595298 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [
[...truncated too long message...]
kout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1375865702
[copy] Copying 241 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1375865702
[delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1375865702
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 39ms :: artifacts dl 2ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 290 minutes 33 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$34.hasMatch(FilePath.java:2678)
at hudson.FilePath$34.invoke(FilePath.java:2557)
at hudson.FilePath$34.invoke(FilePath.java:2547)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1036)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1038)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 363 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/363/
1 tests failed.
FAILED: org.apache.solr.cloud.TestCloudPivotFacet.test
Error Message:
{main(facet=true&facet.pivot={!stats%3Dst0}bogus_not_in_any_doc_s&facet.pivot={!stats%3Dst2}pivot_z_s1,pivot_x_s,pivot_tdt&facet.limit=15&facet.missing=true&facet.sort=index&facet.overrequest.count=0),extra(rows=0&q=*:*&stats=true&stats.field={!key%3Dsk1+tag%3Dst1,st2}pivot_d&stats.field={!key%3Dsk2+tag%3Dst2,st3}pivot_dt1&stats.field={!key%3Dsk3+tag%3Dst3,st4}pivot_z_s&_test_miss=true&_test_sort=index)} ==> Stddev of sk2 => pivot_z_s1,pivot_x_s,pivot_tdt: {params(rows=0),defaults({main({main(rows=0&q=*:*&stats=true&stats.field={!key%3Dsk1+tag%3Dst1,st2}pivot_d&stats.field={!key%3Dsk2+tag%3Dst2,st3}pivot_dt1&stats.field={!key%3Dsk3+tag%3Dst3,st4}pivot_z_s&_test_miss=true&_test_sort=index),extra(fq={!term+f%3Dpivot_z_s1}a)}),extra(fq={!term+f%3Dpivot_x_s}b)})} expected:<0.0> but was:<20724.30287367949>
Stack Trace:
java.lang.AssertionError: {main(facet=true&facet.pivot={!stats%3Dst0}bogus_not_in_any_doc_s&facet.pivot={!stats%3Dst2}pivot_z_s1,pivot_x_s,pivot_tdt&facet.limit=15&facet.missing=true&facet.sort=index&facet.overrequest.count=0),extra(rows=0&q=*:*&stats=true&stats.field={!key%3Dsk1+tag%3Dst1,st2}pivot_d&stats.field={!key%3Dsk2+tag%3Dst2,st3}pivot_dt1&stats.field={!key%3Dsk3+tag%3Dst3,st4}pivot_z_s&_test_miss=true&_test_sort=index)} ==> Stddev of sk2 => pivot_z_s1,pivot_x_s,pivot_tdt: {params(rows=0),defaults({main({main(rows=0&q=*:*&stats=true&stats.field={!key%3Dsk1+tag%3Dst1,st2}pivot_d&stats.field={!key%3Dsk2+tag%3Dst2,st3}pivot_dt1&stats.field={!key%3Dsk3+tag%3Dst3,st4}pivot_z_s&_test_miss=true&_test_sort=index),extra(fq={!term+f%3Dpivot_z_s1}a)}),extra(fq={!term+f%3Dpivot_x_s}b)})} expected:<0.0> but was:<20724.30287367949>
at __randomizedtesting.SeedInfo.seed([E8745FA6D96CE260:6020607C77908F98]:0)
at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:292)
at org.apache.solr.cloud.TestCloudPivotFacet.test(TestCloudPivotFacet.java:239)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1010)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:985)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.AssertionError: Stddev of sk2 => pivot_z_s1,pivot_x_s,pivot_tdt: {params(rows=0),defaults({main({main(rows=0&q=*:*&stats=true&stats.field={!key%3Dsk1+tag%3Dst1,st2}pivot_d&stats.field={!key%3Dsk2+tag%3Dst2,st3}pivot_dt1&stats.field={!key%3Dsk3+tag%3Dst3,st4}pivot_z_s&_test_miss=true&_test_sort=index),extra(fq={!term+f%3Dpivot_z_s1}a)}),extra(fq={!term+f%3Dpivot_x_s}b)})} expected:<0.0> but was:<20724.30287367949>
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:443)
at org.apache.solr.cloud.TestCloudPivotFacet.assertNumerics(TestCloudPivotFacet.java:743)
at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotStats(TestCloudPivotFacet.java:402)
at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotData(TestCloudPivotFacet.java:350)
at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:313)
at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:324)
at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:282)
... 42 more
Build Log:
[...truncated 14318 lines...]
[junit4] Suite: org.apache.solr.cloud.TestCloudPivotFacet
[junit4] 2> 1969745 INFO (SUITE-TestCloudPivotFacet-seed#[E8745FA6D96CE260]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/init-core-data-001
[junit4] 2> 1969746 WARN (SUITE-TestCloudPivotFacet-seed#[E8745FA6D96CE260]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=5 numCloses=5
[junit4] 2> 1969746 INFO (SUITE-TestCloudPivotFacet-seed#[E8745FA6D96CE260]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 1969748 INFO (SUITE-TestCloudPivotFacet-seed#[E8745FA6D96CE260]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
[junit4] 2> 1969748 INFO (SUITE-TestCloudPivotFacet-seed#[E8745FA6D96CE260]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 1969749 INFO (SUITE-TestCloudPivotFacet-seed#[E8745FA6D96CE260]-worker) [ ] o.a.s.c.TestCloudPivotFacet init'ing useFieldRandomizedFactor = 4
[junit4] 2> 1969755 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1969755 INFO (Thread-3174) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1969755 INFO (Thread-3174) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1969757 ERROR (Thread-3174) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1969855 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkTestServer start zk server on port:43751
[junit4] 2> 1969859 INFO (zkConnectionManagerCallback-3792-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1969864 INFO (zkConnectionManagerCallback-3794-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1969872 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1969874 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 1969876 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1969878 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1969879 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1969881 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1969882 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1969884 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1969885 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1969887 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1969888 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1969890 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 1970067 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 1970069 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1970069 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1970069 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1970069 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@69a7986a{/,null,AVAILABLE}
[junit4] 2> 1970071 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5c0d69{HTTP/1.1,[http/1.1]}{127.0.0.1:46634}
[junit4] 2> 1970071 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.Server Started @1970163ms
[junit4] 2> 1970071 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/tempDir-001/control/data, hostContext=/, hostPort=46634, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/control-001/cores}
[junit4] 2> 1970071 ERROR (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1970071 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1970071 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 1970072 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1970072 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1970072 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-31T09:03:58.705Z
[junit4] 2> 1970074 INFO (zkConnectionManagerCallback-3796-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1970075 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1970075 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/control-001/solr.xml
[junit4] 2> 1970080 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1970080 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1970082 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1970202 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43751/solr
[junit4] 2> 1970203 INFO (zkConnectionManagerCallback-3800-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1970207 INFO (zkConnectionManagerCallback-3802-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1970304 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46634_
[junit4] 2> 1970305 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.c.Overseer Overseer (id=72195968331743236-127.0.0.1:46634_-n_0000000000) starting
[junit4] 2> 1970313 INFO (zkConnectionManagerCallback-3809-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1970317 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43751/solr ready
[junit4] 2> 1970319 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46634_
[junit4] 2> 1970319 INFO (OverseerStateUpdate-72195968331743236-127.0.0.1:46634_-n_0000000000) [n:127.0.0.1:46634_ ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:46634_
[junit4] 2> 1970336 INFO (OverseerStateUpdate-72195968331743236-127.0.0.1:46634_-n_0000000000) [n:127.0.0.1:46634_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1970341 INFO (zkCallback-3808-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1970358 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1970386 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1970403 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1970403 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1970405 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/control-001/cores
[junit4] 2> 1970438 INFO (zkConnectionManagerCallback-3815-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1970439 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1970440 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43751/solr ready
[junit4] 2> 1970442 INFO (qtp2043184068-14033) [n:127.0.0.1:46634_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:46634_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1970446 INFO (OverseerThreadFactory-5492-thread-1-processing-n:127.0.0.1:46634_) [n:127.0.0.1:46634_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 1970556 INFO (qtp2043184068-14028) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1970564 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1970564 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1971608 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 1971635 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 1971828 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1971853 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 1971854 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1971854 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 1971859 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=35.2265625, floorSegmentMB=0.6015625, forceMergeDeletesPctAllowed=14.107885589266665, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=27.694887829024218
[junit4] 2> 1971865 WARN (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1971950 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1971950 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1971952 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1971952 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1971955 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=6, maxMergeAtOnceExplicit=9, maxMergedSegmentMB=0.7294921875, floorSegmentMB=0.921875, forceMergeDeletesPctAllowed=7.396377397787985, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7317311934972269, deletesPctAllowed=48.98606555317808
[junit4] 2> 1971957 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@352f4d3d[control_collection_shard1_replica_n1] main]
[junit4] 2> 1971959 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1971959 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1971960 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1971962 INFO (searcherExecutor-5497-thread-1-processing-n:127.0.0.1:46634_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@352f4d3d[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1971968 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615831121894834176
[junit4] 2> 1971976 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 1971979 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1971979 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1971980 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:46634/control_collection_shard1_replica_n1/
[junit4] 2> 1971980 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1971980 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:46634/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 1971980 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1971984 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:46634/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 1972139 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1972142 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1578
[junit4] 2> 1972145 INFO (qtp2043184068-14033) [n:127.0.0.1:46634_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1972242 INFO (zkCallback-3801-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1972449 INFO (OverseerCollectionConfigSetProcessor-72195968331743236-127.0.0.1:46634_-n_0000000000) [n:127.0.0.1:46634_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1973145 INFO (qtp2043184068-14033) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:46634_&wt=javabin&version=2} status=0 QTime=2703
[junit4] 2> 1973152 INFO (zkConnectionManagerCallback-3820-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1973153 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1973154 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43751/solr ready
[junit4] 2> 1973154 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1973155 INFO (qtp2043184068-14032) [n:127.0.0.1:46634_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1973158 INFO (OverseerThreadFactory-5492-thread-2-processing-n:127.0.0.1:46634_) [n:127.0.0.1:46634_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 1973364 WARN (OverseerThreadFactory-5492-thread-2-processing-n:127.0.0.1:46634_) [n:127.0.0.1:46634_ ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 1973367 INFO (qtp2043184068-14032) [n:127.0.0.1:46634_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1973367 INFO (qtp2043184068-14032) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=211
[junit4] 2> 1973549 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-1-001 of type NRT
[junit4] 2> 1973550 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 1973551 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1973551 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1973551 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1973552 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4e05206a{/,null,AVAILABLE}
[junit4] 2> 1973553 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6fc5b35c{HTTP/1.1,[http/1.1]}{127.0.0.1:33700}
[junit4] 2> 1973553 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.Server Started @1973646ms
[junit4] 2> 1973553 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=33700, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-1-001/cores}
[junit4] 2> 1973554 ERROR (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1973554 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1973554 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 1973554 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1973554 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1973554 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-31T09:04:02.187Z
[junit4] 2> 1973556 INFO (zkConnectionManagerCallback-3822-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1973557 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1973557 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-1-001/solr.xml
[junit4] 2> 1973562 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1973562 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1973564 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1973928 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43751/solr
[junit4] 2> 1973930 INFO (zkConnectionManagerCallback-3826-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1973932 INFO (zkConnectionManagerCallback-3828-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1973941 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1973946 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1973946 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33700_
[junit4] 2> 1973947 INFO (zkCallback-3808-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1973947 INFO (zkCallback-3801-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1973947 INFO (zkCallback-3819-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1973948 INFO (zkCallback-3827-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1973977 INFO (zkConnectionManagerCallback-3835-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1973978 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1973979 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43751/solr ready
[junit4] 2> 1973980 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1974009 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1974027 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1974027 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1974030 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-1-001/cores
[junit4] 2> 1974073 INFO (qtp502179821-14088) [n:127.0.0.1:33700_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:33700_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1974076 INFO (OverseerCollectionConfigSetProcessor-72195968331743236-127.0.0.1:46634_-n_0000000000) [n:127.0.0.1:46634_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1974081 INFO (qtp502179821-14083) [n:127.0.0.1:33700_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=2
[junit4] 2> 1974085 INFO (qtp2043184068-14028) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=2
[junit4] 2> 1974086 INFO (qtp2043184068-14032) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
[junit4] 2> 1974087 INFO (OverseerThreadFactory-5492-thread-3-processing-n:127.0.0.1:46634_) [n:127.0.0.1:46634_ c:collection1 s:shard2 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:33700_ for creating new replica of shard shard2 for collection collection1
[junit4] 2> 1974092 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ x:collection1_shard2_replica_n41] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1975112 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 1975129 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema [collection1_shard2_replica_n41] Schema name=test
[junit4] 2> 1975280 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1975298 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n41' using configuration from collection collection1, trusted=true
[junit4] 2> 1975299 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n41' (registry 'solr.core.collection1.shard2.replica_n41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1975299 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [[collection1_shard2_replica_n41] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-1-001/cores/collection1_shard2_replica_n41], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-1-001/cores/collection1_shard2_replica_n41/data/]
[junit4] 2> 1975306 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=35.2265625, floorSegmentMB=0.6015625, forceMergeDeletesPctAllowed=14.107885589266665, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=27.694887829024218
[junit4] 2> 1975310 WARN (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1975383 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1975383 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1975385 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1975385 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1975387 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=6, maxMergeAtOnceExplicit=9, maxMergedSegmentMB=0.7294921875, floorSegmentMB=0.921875, forceMergeDeletesPctAllowed=7.396377397787985, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7317311934972269, deletesPctAllowed=48.98606555317808
[junit4] 2> 1975388 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.SolrIndexSearcher Opening [Searcher@3f1ba90e[collection1_shard2_replica_n41] main]
[junit4] 2> 1975389 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1975390 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1975391 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1975392 INFO (searcherExecutor-5511-thread-1-processing-n:127.0.0.1:33700_ x:collection1_shard2_replica_n41 c:collection1 s:shard2) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [collection1_shard2_replica_n41] Registered new searcher Searcher@3f1ba90e[collection1_shard2_replica_n41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1975392 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615831125485158400
[junit4] 2> 1975399 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard2 to Terms{values={core_node42=0}, version=0}
[junit4] 2> 1975402 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1975402 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1975402 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:33700/collection1_shard2_replica_n41/
[junit4] 2> 1975402 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1975403 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy http://127.0.0.1:33700/collection1_shard2_replica_n41/ has no replicas
[junit4] 2> 1975403 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1975407 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:33700/collection1_shard2_replica_n41/ shard2
[junit4] 2> 1975559 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1975562 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1471
[junit4] 2> 1975566 INFO (qtp502179821-14088) [n:127.0.0.1:33700_ c:collection1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:33700_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1492
[junit4] 2> 1975753 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-2-001 of type NRT
[junit4] 2> 1975755 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 1975757 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1975757 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1975757 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1975758 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7bfd2e5c{/,null,AVAILABLE}
[junit4] 2> 1975760 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4947e03d{HTTP/1.1,[http/1.1]}{127.0.0.1:37293}
[junit4] 2> 1975760 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.Server Started @1975853ms
[junit4] 2> 1975760 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/tempDir-001/jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=37293, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-2-001/cores}
[junit4] 2> 1975760 ERROR (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1975761 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1975761 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 1975761 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1975761 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1975761 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-31T09:04:04.394Z
[junit4] 2> 1975763 INFO (zkConnectionManagerCallback-3838-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1975764 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1975764 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-2-001/solr.xml
[junit4] 2> 1975769 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1975769 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1975771 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1975977 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43751/solr
[junit4] 2> 1975979 INFO (zkConnectionManagerCallback-3842-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1975982 INFO (zkConnectionManagerCallback-3844-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1975989 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1975994 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1975994 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37293_
[junit4] 2> 1975996 INFO (zkCallback-3834-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1975996 INFO (zkCallback-3808-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1975996 INFO (zkCallback-3819-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1975996 INFO (zkCallback-3801-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1975996 INFO (zkCallback-3827-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1975997 INFO (zkCallback-3843-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1976025 INFO (zkConnectionManagerCallback-3851-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1976026 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1976027 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43751/solr ready
[junit4] 2> 1976028 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1976057 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1976075 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1976075 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1976078 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-2-001/cores
[junit4] 2> 1976079 INFO (OverseerCollectionConfigSetProcessor-72195968331743236-127.0.0.1:46634_-n_0000000000) [n:127.0.0.1:46634_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1976134 INFO (qtp502179821-14090) [n:127.0.0.1:33700_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:37293_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1976140 INFO (qtp502179821-14087) [n:127.0.0.1:33700_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.collection1.shard2.replica_n41:INDEX.sizeInBytes} status=0 QTime=1
[junit4] 2> 1976142 INFO (qtp502179821-14083) [n:127.0.0.1:33700_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1976144 INFO (qtp490614089-14125) [n:127.0.0.1:37293_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1976145 INFO (qtp2043184068-14028) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 1976146 INFO (qtp2043184068-14032) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1976146 INFO (OverseerThreadFactory-5492-thread-4-processing-n:127.0.0.1:46634_) [n:127.0.0.1:46634_ c:collection1 s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:37293_ for creating new replica of shard shard1 for collection collection1
[junit4] 2> 1976149 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ x:collection1_shard1_replica_n43] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1977168 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 1977186 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.IndexSchema [collection1_shard1_replica_n43] Schema name=test
[junit4] 2> 1977328 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1977351 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n43' using configuration from collection collection1, trusted=true
[junit4] 2> 1977352 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n43' (registry 'solr.core.collection1.shard1.replica_n43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1977353 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrCore [[collection1_shard1_replica_n43] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-2-001/cores/collection1_shard1_replica_n43], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-2-001/cores/collection1_shard1_replica_n43/data/]
[junit4] 2> 1977358 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=35.2265625, floorSegmentMB=0.6015625, forceMergeDeletesPctAllowed=14.107885589266665, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=27.694887829024218
[junit4] 2> 1977362 WARN (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1977434 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1977434 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1977436 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1977436 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1977438 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=6, maxMergeAtOnceExplicit=9, maxMergedSegmentMB=0.7294921875, floorSegmentMB=0.921875, forceMergeDeletesPctAllowed=7.396377397787985, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7317311934972269, deletesPctAllowed=48.98606555317808
[junit4] 2> 1977439 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.SolrIndexSearcher Opening [Searcher@35041ba[collection1_shard1_replica_n43] main]
[junit4] 2> 1977440 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1977441 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1977442 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1977442 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615831127634739200
[junit4] 2> 1977443 INFO (searcherExecutor-5525-thread-1-processing-n:127.0.0.1:37293_ x:collection1_shard1_replica_n43 c:collection1 s:shard1) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrCore [collection1_shard1_replica_n43] Registered new searcher Searcher@35041ba[collection1_shard1_replica_n43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1977447 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node44=0}, version=0}
[junit4] 2> 1977450 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1977450 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1977450 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:37293/collection1_shard1_replica_n43/
[junit4] 2> 1977450 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1977450 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy http://127.0.0.1:37293/collection1_shard1_replica_n43/ has no replicas
[junit4] 2> 1977450 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1977454 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:37293/collection1_shard1_replica_n43/ shard1
[junit4] 2> 1977557 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1977562 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1412
[junit4] 2> 1977564 INFO (qtp502179821-14090) [n:127.0.0.1:33700_ c:collection1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:37293_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1430
[junit4] 2> 1977745 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-3-001 of type NRT
[junit4] 2> 1977746 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 1977748 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1977748 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1977748 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 1977748 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6489f990{/,null,AVAILABLE}
[junit4] 2> 1977749 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6b364fcb{HTTP/1.1,[http/1.1]}{127.0.0.1:38180}
[junit4] 2> 1977749 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.Server Started @1977841ms
[junit4] 2> 1977749 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/tempDir-001/jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=38180, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-3-001/cores}
[junit4] 2> 1977749 ERROR (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1977749 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1977749 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 1977749 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1977749 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1977749 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-31T09:04:06.382Z
[junit4] 2> 1977754 INFO (zkConnectionManagerCallback-3854-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1977755 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1977755 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-3-001/solr.xml
[junit4] 2> 1977760 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1977760 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1977762 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1977878 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43751/solr
[junit4] 2> 1977880 INFO (zkConnectionManagerCallback-3858-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1977883 INFO (zkConnectionManagerCallback-3860-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1977890 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1977900 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1977900 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38180_
[junit4] 2> 1977901 INFO (zkCallback-3819-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1977902 INFO (zkCallback-3843-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1977902 INFO (zkCallback-3808-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1977902 INFO (zkCallback-3834-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1977902 INFO (zkCallback-3801-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1977902 INFO (zkCallback-3850-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1977903 INFO (zkCallback-3859-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1977903 INFO (zkCallback-3827-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1977934 INFO (zkConnectionManagerCallback-3867-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1977935 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 1977937 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43751/solr ready
[junit4] 2> 1977937 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1977967 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1977985 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1977986 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1977988 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-3-001/cores
[junit4] 2> 1978078 INFO (qtp502179821-14089) [n:127.0.0.1:33700_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:38180_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1978081 INFO (OverseerCollectionConfigSetProcessor-72195968331743236-127.0.0.1:46634_-n_0000000000) [n:127.0.0.1:46634_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1978084 INFO (qtp502179821-14087) [n:127.0.0.1:33700_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.collection1.shard2.replica_n41:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 1978085 INFO (qtp502179821-14083) [n:127.0.0.1:33700_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1978088 INFO (qtp490614089-14121) [n:127.0.0.1:37293_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.collection1.shard1.replica_n43:INDEX.sizeInBytes} status=0 QTime=1
[junit4] 2> 1978089 INFO (qtp490614089-14127) [n:127.0.0.1:37293_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1978091 INFO (qtp643415226-14162) [n:127.0.0.1:38180_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1978092 INFO (qtp2043184068-14028) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 1978093 INFO (qtp2043184068-14032) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1978095 INFO (qtp502179821-14087) [n:127.0.0.1:33700_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.collection1.shard2.replica_n41:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 1978096 INFO (OverseerThreadFactory-5492-thread-5-processing-n:127.0.0.1:46634_) [n:127.0.0.1:46634_ c:collection1 s:shard2 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:38180_ for creating new replica of shard shard2 for collection collection1
[junit4] 2> 1978099 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ x:collection1_shard2_replica_n45] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1979125 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 1979141 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.s.IndexSchema [collection1_shard2_replica_n45] Schema name=test
[junit4] 2> 1979293 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1979314 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n45' using configuration from collection collection1, trusted=true
[junit4] 2> 1979323 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n45' (registry 'solr.core.collection1.shard2.replica_n45') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1979324 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.SolrCore [[collection1_shard2_replica_n45] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-3-001/cores/collection1_shard2_replica_n45], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-3-001/cores/collection1_shard2_replica_n45/data/]
[junit4] 2> 1979329 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=35.2265625, floorSegmentMB=0.6015625, forceMergeDeletesPctAllowed=14.107885589266665, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=27.694887829024218
[junit4] 2> 1979333 WARN (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1979405 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1979405 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1979407 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1979407 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1979409 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=6, maxMergeAtOnceExplicit=9, maxMergedSegmentMB=0.7294921875, floorSegmentMB=0.921875, forceMergeDeletesPctAllowed=7.396377397787985, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7317311934972269, deletesPctAllowed=48.98606555317808
[junit4] 2> 1979410 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.s.SolrIndexSearcher Opening [Searcher@711642ac[collection1_shard2_replica_n45] main]
[junit4] 2> 1979412 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1979412 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1979413 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1979413 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615831129701482496
[junit4] 2> 1979415 INFO (searcherExecutor-5539-thread-1-processing-n:127.0.0.1:38180_ x:collection1_shard2_replica_n45 c:collection1 s:shard2) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.SolrCore [collection1_shard2_replica_n45] Registered new searcher Searcher@711642ac[collection1_shard2_replica_n45] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1979422 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard2 to Terms{values={core_node42=0, core_node46=0}, version=1}
[junit4] 2> 1979424 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.ZkController Core needs to recover:collection1_shard2_replica_n45
[junit4] 2> 1979424 INFO (updateExecutor-3855-thread-1-processing-n:127.0.0.1:38180_ x:collection1_shard2_replica_n45 c:collection1 s:shard2) [n:127.0.0.1:38180_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 1979425 INFO (recoveryExecutor-3856-thread-1-processing-n:127.0.0.1:38180_ x:collection1_shard2_replica_n45 c:collection1 s:shard2 r:core_node46) [n:127.0.0.1:38180_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 1979426 INFO (recoveryExecutor-3856-thread-1-processing-n:127.0.0.1:38180_ x:collection1_shard2_replica_n45 c:collection1 s:shard2 r:core_node46) [n:127.0.0.1:38180_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.c.RecoveryStrategy startupVersions is empty
[junit4] 2> 1979427 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/co
[...truncated too long message...]
kout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1821347088
[copy] Copying 241 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1821347088
[delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1821347088
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 49ms :: artifacts dl 4ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 282 minutes 36 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$34.hasMatch(FilePath.java:2678)
at hudson.FilePath$34.invoke(FilePath.java:2557)
at hudson.FilePath$34.invoke(FilePath.java:2547)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1036)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1038)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 362 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/362/
3 tests failed.
FAILED: org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest.test
Error Message:
There are still nodes recoverying - waited for 320 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
at __randomizedtesting.SeedInfo.seed([949909B0C215D49:811DAF41A2DD30B1]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:920)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1477)
at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:145)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1010)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:985)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
Error Message:
7 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest: 1) Thread[id=5723, name=searcherExecutor-2132-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=5686, name=searcherExecutor-2118-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=5766, name=searcherExecutor-2146-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=5904, name=searcherExecutor-2180-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=5644, name=searcherExecutor-2104-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=5869, name=searcherExecutor-2166-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=5941, name=searcherExecutor-2194-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 7 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest:
1) Thread[id=5723, name=searcherExecutor-2132-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=5686, name=searcherExecutor-2118-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=5766, name=searcherExecutor-2146-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=5904, name=searcherExecutor-2180-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=5644, name=searcherExecutor-2104-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=5869, name=searcherExecutor-2166-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=5941, name=searcherExecutor-2194-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([949909B0C215D49]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=5723, name=searcherExecutor-2132-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=5686, name=searcherExecutor-2118-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=5766, name=searcherExecutor-2146-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=5904, name=searcherExecutor-2180-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=5644, name=searcherExecutor-2104-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=5869, name=searcherExecutor-2166-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=5941, name=searcherExecutor-2194-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=5723, name=searcherExecutor-2132-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=5686, name=searcherExecutor-2118-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=5766, name=searcherExecutor-2146-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=5904, name=searcherExecutor-2180-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=5644, name=searcherExecutor-2104-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=5869, name=searcherExecutor-2166-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=5941, name=searcherExecutor-2194-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([949909B0C215D49]:0)
Build Log:
[...truncated 13860 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/init-core-data-001
[junit4] 2> 779753 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=57 numCloses=57
[junit4] 2> 779753 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 779755 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 779755 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 779755 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /_m/xy
[junit4] 2> 781737 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 784178 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 784871 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log Logging to org.apache.logging.slf4j.Log4jLogger@3c686729 via org.mortbay.log.Slf4jLog
[junit4] 2> 784924 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 785806 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 785928 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_39760_hdfs____.gugpqj/webapp
[junit4] 2> 787522 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39760
[junit4] 2> 789386 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 789393 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 789420 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_38162_datanode____uc2b67/webapp
[junit4] 2> 789895 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:38162
[junit4] 2> 791647 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 791648 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 791689 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_35321_datanode____1ri1h3/webapp
[junit4] 2> 792959 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35321
[junit4] 2> 795242 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:39808) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 795243 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:39808) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 795962 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x7173ecbb8897a: from storage DS-3a2efac9-5dc3-49a8-897c-356737a8a8b0 node DatanodeRegistration(127.0.0.1:44434, datanodeUuid=6c94df5d-b015-4457-96ef-c82bc84563ac, infoPort=40113, infoSecurePort=0, ipcPort=39705, storageInfo=lv=-56;cid=testClusterID;nsid=1544823971;c=0), blocks: 0, hasStaleStorage: true, processing time: 9 msecs
[junit4] 2> 795963 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x7173ecbe66b90: from storage DS-4ddac0f4-88bd-4a15-abe5-c5925c828a6a node DatanodeRegistration(127.0.0.1:37765, datanodeUuid=56a425b1-8fa9-48b5-9907-4c713cae058b, infoPort=35248, infoSecurePort=0, ipcPort=45180, storageInfo=lv=-56;cid=testClusterID;nsid=1544823971;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 795963 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x7173ecbb8897a: from storage DS-d14b8728-3e26-4b62-ad71-642792614414 node DatanodeRegistration(127.0.0.1:44434, datanodeUuid=6c94df5d-b015-4457-96ef-c82bc84563ac, infoPort=40113, infoSecurePort=0, ipcPort=39705, storageInfo=lv=-56;cid=testClusterID;nsid=1544823971;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 795969 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x7173ecbe66b90: from storage DS-672b6f93-849d-4a48-a7ea-4f9c043c05ea node DatanodeRegistration(127.0.0.1:37765, datanodeUuid=56a425b1-8fa9-48b5-9907-4c713cae058b, infoPort=35248, infoSecurePort=0, ipcPort=45180, storageInfo=lv=-56;cid=testClusterID;nsid=1544823971;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 797674 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 797674 INFO (Thread-1656) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 797674 INFO (Thread-1656) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 797677 ERROR (Thread-1656) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 797774 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.ZkTestServer start zk server on port:41780
[junit4] 2> 797793 INFO (zkConnectionManagerCallback-1083-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 797813 INFO (zkConnectionManagerCallback-1085-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 797831 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 797839 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 797841 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 797843 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 797845 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 797848 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 797858 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 797865 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 797874 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 797882 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 797892 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 797896 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 798573 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 798575 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 798575 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 798575 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 798577 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4327f207{/_m/xy,null,AVAILABLE}
[junit4] 2> 798578 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@21c29571{HTTP/1.1,[http/1.1]}{127.0.0.1:37095}
[junit4] 2> 798578 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.Server Started @798773ms
[junit4] 2> 798578 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:39808/hdfs__localhost_39808__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001_tempDir-002_control_data, replicaType=NRT, hostContext=/_m/xy, hostPort=37095, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/control-001/cores}
[junit4] 2> 798579 ERROR (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 798605 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 798605 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 798606 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 798606 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 798606 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-30T00:58:20.248Z
[junit4] 2> 798652 INFO (zkConnectionManagerCallback-1087-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 798663 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 798663 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/control-001/solr.xml
[junit4] 2> 798691 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 798691 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 798693 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 799227 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41780/solr
[junit4] 2> 799250 INFO (zkConnectionManagerCallback-1091-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 799255 INFO (zkConnectionManagerCallback-1093-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 799555 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37095__m%2Fxy
[junit4] 2> 799557 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.Overseer Overseer (id=72188396363120644-127.0.0.1:37095__m%2Fxy-n_0000000000) starting
[junit4] 2> 799635 INFO (zkConnectionManagerCallback-1100-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 799646 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41780/solr ready
[junit4] 2> 799656 INFO (OverseerStateUpdate-72188396363120644-127.0.0.1:37095__m%2Fxy-n_0000000000) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:37095__m%2Fxy
[junit4] 2> 799684 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37095__m%2Fxy
[junit4] 2> 799697 INFO (zkCallback-1092-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 799715 INFO (zkCallback-1099-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 799725 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 799777 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 799875 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 799875 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 799878 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/control-001/cores
[junit4] 2> 799933 INFO (zkConnectionManagerCallback-1106-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 799934 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 799935 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41780/solr ready
[junit4] 2> 799937 INFO (qtp731027699-5604) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:37095__m%252Fxy&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 799955 INFO (OverseerThreadFactory-2099-thread-1-processing-n:127.0.0.1:37095__m%2Fxy) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 800193 INFO (qtp731027699-5599) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 800221 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 800222 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 801303 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 801430 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 803814 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 803875 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 803876 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 803877 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 803917 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=21.619140625, floorSegmentMB=0.26953125, forceMergeDeletesPctAllowed=22.12888844486297, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6395981875322819, deletesPctAllowed=45.80989708961823
[junit4] 2> 803959 WARN (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 804245 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 804245 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 804247 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 30000ms;
[junit4] 2> 804247 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
[junit4] 2> 804251 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=88.462890625, floorSegmentMB=1.7255859375, forceMergeDeletesPctAllowed=24.202588505252482, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6912498608789626, deletesPctAllowed=31.804024835151772
[junit4] 2> 804252 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@27e94675[control_collection_shard1_replica_n1] main]
[junit4] 2> 804260 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 804262 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 804264 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 804265 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615709974987538432
[junit4] 2> 804267 INFO (searcherExecutor-2104-thread-1-processing-n:127.0.0.1:37095__m%2Fxy x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@27e94675[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 804293 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 804316 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 804316 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 804316 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:37095/_m/xy/control_collection_shard1_replica_n1/
[junit4] 2> 804317 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 804317 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:37095/_m/xy/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 804317 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 804325 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:37095/_m/xy/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 804379 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 804383 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=4161
[junit4] 2> 804390 INFO (qtp731027699-5604) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 804482 INFO (zkCallback-1092-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 805390 INFO (qtp731027699-5604) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:37095__m%252Fxy&wt=javabin&version=2} status=0 QTime=5453
[junit4] 2> 805423 INFO (zkConnectionManagerCallback-1111-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 805426 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 805427 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41780/solr ready
[junit4] 2> 805434 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 805437 INFO (qtp731027699-5599) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 805450 INFO (OverseerThreadFactory-2099-thread-2-processing-n:127.0.0.1:37095__m%2Fxy) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 805450 INFO (OverseerCollectionConfigSetProcessor-72188396363120644-127.0.0.1:37095__m%2Fxy-n_0000000000) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 805720 WARN (OverseerThreadFactory-2099-thread-2-processing-n:127.0.0.1:37095__m%2Fxy) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 805724 INFO (qtp731027699-5599) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 805724 INFO (qtp731027699-5599) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=287
[junit4] 2> 806100 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-1-001 of type NRT
[junit4] 2> 806104 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 806105 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 806105 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 806106 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 806106 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6e6b8bac{/_m/xy,null,AVAILABLE}
[junit4] 2> 806107 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@234f6dae{HTTP/1.1,[http/1.1]}{127.0.0.1:45324}
[junit4] 2> 806107 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.Server Started @806302ms
[junit4] 2> 806107 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:39808/hdfs__localhost_39808__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001_tempDir-002_jetty1, solrconfig=solrconfig.xml, hostContext=/_m/xy, hostPort=45324, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-1-001/cores}
[junit4] 2> 806107 ERROR (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 806108 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 806108 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 806108 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 806108 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 806108 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-30T00:58:27.750Z
[junit4] 2> 806116 INFO (zkConnectionManagerCallback-1113-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 806121 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 806121 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-1-001/solr.xml
[junit4] 2> 806127 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 806127 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 806129 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 806468 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41780/solr
[junit4] 2> 806500 INFO (zkConnectionManagerCallback-1117-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 806506 INFO (zkConnectionManagerCallback-1119-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 806519 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 806526 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 806526 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45324__m%2Fxy
[junit4] 2> 806527 INFO (zkCallback-1092-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 806527 INFO (zkCallback-1110-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 806527 INFO (zkCallback-1099-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 806544 INFO (zkCallback-1118-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 806567 INFO (zkConnectionManagerCallback-1126-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 806569 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 806571 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41780/solr ready
[junit4] 2> 806571 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 806618 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 806660 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 806660 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 806665 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-1-001/cores
[junit4] 2> 806729 INFO (qtp731027699-5603) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:45324__m%252Fxy&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 806736 INFO (OverseerCollectionConfigSetProcessor-72188396363120644-127.0.0.1:37095__m%2Fxy-n_0000000000) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 806745 INFO (qtp731027699-5605) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=1
[junit4] 2> 806748 INFO (qtp731027699-5604) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
[junit4] 2> 806754 INFO (qtp2137760111-5658) [n:127.0.0.1:45324__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 806762 INFO (OverseerThreadFactory-2099-thread-3-processing-n:127.0.0.1:37095__m%2Fxy) [n:127.0.0.1:37095__m%2Fxy c:collection1 s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:45324__m%2Fxy for creating new replica of shard shard1 for collection collection1
[junit4] 2> 806784 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy x:collection1_shard1_replica_n21] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 807845 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 807880 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema [collection1_shard1_replica_n21] Schema name=test
[junit4] 2> 808055 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 808111 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n21' using configuration from collection collection1, trusted=true
[junit4] 2> 808114 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n21' (registry 'solr.core.collection1.shard1.replica_n21') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 808114 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [[collection1_shard1_replica_n21] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-1-001/cores/collection1_shard1_replica_n21], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-1-001/cores/collection1_shard1_replica_n21/data/]
[junit4] 2> 808142 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=21.619140625, floorSegmentMB=0.26953125, forceMergeDeletesPctAllowed=22.12888844486297, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6395981875322819, deletesPctAllowed=45.80989708961823
[junit4] 2> 808155 WARN (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 808331 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 808331 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 808333 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 30000ms;
[junit4] 2> 808333 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
[junit4] 2> 808337 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=88.462890625, floorSegmentMB=1.7255859375, forceMergeDeletesPctAllowed=24.202588505252482, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6912498608789626, deletesPctAllowed=31.804024835151772
[junit4] 2> 808338 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.SolrIndexSearcher Opening [Searcher@6b9d1fc4[collection1_shard1_replica_n21] main]
[junit4] 2> 808346 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 808347 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 808351 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 808351 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615709979272019968
[junit4] 2> 808361 INFO (searcherExecutor-2118-thread-1-processing-n:127.0.0.1:45324__m%2Fxy x:collection1_shard1_replica_n21 c:collection1 s:shard1) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [collection1_shard1_replica_n21] Registered new searcher Searcher@6b9d1fc4[collection1_shard1_replica_n21] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 808578 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node22=0}, version=0}
[junit4] 2> 808582 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 808582 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 808582 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45324/_m/xy/collection1_shard1_replica_n21/
[junit4] 2> 808582 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 808582 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy http://127.0.0.1:45324/_m/xy/collection1_shard1_replica_n21/ has no replicas
[junit4] 2> 808582 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 808587 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45324/_m/xy/collection1_shard1_replica_n21/ shard1
[junit4] 2> 808739 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 808745 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1961
[junit4] 2> 808749 INFO (qtp731027699-5603) [n:127.0.0.1:37095__m%2Fxy c:collection1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:45324__m%252Fxy&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=2019
[junit4] 2> 809239 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-2-001 of type NRT
[junit4] 2> 809242 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 809301 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 809301 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 809301 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 809301 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@459cec24{/_m/xy,null,AVAILABLE}
[junit4] 2> 809302 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@381a485{HTTP/1.1,[http/1.1]}{127.0.0.1:40664}
[junit4] 2> 809302 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.Server Started @809497ms
[junit4] 2> 809302 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:39808/hdfs__localhost_39808__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001_tempDir-002_jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/_m/xy, hostPort=40664, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-2-001/cores}
[junit4] 2> 809303 ERROR (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 809307 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 809307 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 809307 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 809307 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 809307 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-30T00:58:30.949Z
[junit4] 2> 809338 INFO (zkConnectionManagerCallback-1129-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 809340 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 809340 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-2-001/solr.xml
[junit4] 2> 809346 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 809346 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 809347 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 809727 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41780/solr
[junit4] 2> 809738 INFO (zkConnectionManagerCallback-1133-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 809744 INFO (zkConnectionManagerCallback-1135-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 809755 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 809764 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 809764 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40664__m%2Fxy
[junit4] 2> 809768 INFO (zkCallback-1092-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 809768 INFO (zkCallback-1110-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 809768 INFO (zkCallback-1099-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 809768 INFO (zkCallback-1118-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 809768 INFO (zkCallback-1125-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 809795 INFO (zkCallback-1134-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 809818 INFO (zkConnectionManagerCallback-1142-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 809820 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 809823 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41780/solr ready
[junit4] 2> 809824 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 809860 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 809881 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 809881 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 809884 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-2-001/cores
[junit4] 2> 809941 INFO (qtp731027699-5604) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:40664__m%252Fxy&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 809952 INFO (OverseerCollectionConfigSetProcessor-72188396363120644-127.0.0.1:37095__m%2Fxy-n_0000000000) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 809968 INFO (qtp731027699-5603) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 809970 INFO (qtp731027699-5602) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 809985 INFO (qtp1985293797-5696) [n:127.0.0.1:40664__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 809988 INFO (qtp2137760111-5654) [n:127.0.0.1:45324__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.collection1.shard1.replica_n21:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 809993 INFO (qtp213776
[...truncated too long message...]
kout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1299618971
[copy] Copying 241 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1299618971
[delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1299618971
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 48ms :: artifacts dl 2ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 250 minutes 10 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$34.hasMatch(FilePath.java:2678)
at hudson.FilePath$34.invoke(FilePath.java:2557)
at hudson.FilePath$34.invoke(FilePath.java:2547)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1036)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1038)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)