You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/05/11 18:28:31 UTC
[JENKINS] Lucene-Solr-NightlyTests-7.3 - Build # 24 - Still
unstable
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.3/24/
5 tests failed.
FAILED: org.apache.solr.cloud.RestartWhileUpdatingTest.test
Error Message:
There are still nodes recoverying - waited for 320 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
at __randomizedtesting.SeedInfo.seed([A403300B1438A1D2:2C570FD1BAC4CC2A]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:921)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1478)
at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:144)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.RestartWhileUpdatingTest
Error Message:
7 threads leaked from SUITE scope at org.apache.solr.cloud.RestartWhileUpdatingTest: 1) Thread[id=82432, name=searcherExecutor-2841-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=82606, name=searcherExecutor-2891-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=82639, name=searcherExecutor-2902-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=82467, name=searcherExecutor-2852-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=82393, name=searcherExecutor-2830-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=82573, name=searcherExecutor-2880-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=82509, name=searcherExecutor-2863-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 7 threads leaked from SUITE scope at org.apache.solr.cloud.RestartWhileUpdatingTest:
1) Thread[id=82432, name=searcherExecutor-2841-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=82606, name=searcherExecutor-2891-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=82639, name=searcherExecutor-2902-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=82467, name=searcherExecutor-2852-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=82393, name=searcherExecutor-2830-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=82573, name=searcherExecutor-2880-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=82509, name=searcherExecutor-2863-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([A403300B1438A1D2]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.RestartWhileUpdatingTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=82432, name=searcherExecutor-2841-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=82606, name=searcherExecutor-2891-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=82639, name=searcherExecutor-2902-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=82467, name=searcherExecutor-2852-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=82393, name=searcherExecutor-2830-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=82573, name=searcherExecutor-2880-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=82509, name=searcherExecutor-2863-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=82432, name=searcherExecutor-2841-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=82606, name=searcherExecutor-2891-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=82639, name=searcherExecutor-2902-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=82467, name=searcherExecutor-2852-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=82393, name=searcherExecutor-2830-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=82573, name=searcherExecutor-2880-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=82509, name=searcherExecutor-2863-thread-1, state=WAITING, group=TGRP-RestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([A403300B1438A1D2]:0)
FAILED: org.apache.solr.cloud.api.collections.CustomCollectionTest.testCustomCollectionsAPI
Error Message:
Stack Trace:
java.lang.NullPointerException
at __randomizedtesting.SeedInfo.seed([A403300B1438A1D2:CEE2BE6029A217AA]:0)
at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
at org.apache.solr.cloud.api.collections.CustomCollectionTest.testCustomCollectionsAPI(CustomCollectionTest.java:105)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.client.solrj.io.sql.JdbcTest
Error Message:
Could not load collection from ZK: collection1_collection
Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: collection1_collection
at __randomizedtesting.SeedInfo.seed([AA2728CD153CBC80]:0)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1237)
at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:675)
at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:148)
at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:131)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:154)
at org.apache.solr.client.solrj.io.sql.JdbcTest.setupCluster(JdbcTest.java:78)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1_collection/state.json
at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1249)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1235)
... 29 more
Build Log:
[...truncated 13260 lines...]
[junit4] Suite: org.apache.solr.cloud.api.collections.CustomCollectionTest
[junit4] 2> 524451 INFO (SUITE-CustomCollectionTest-seed#[A403300B1438A1D2]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.3/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CustomCollectionTest_A403300B1438A1D2-001/init-core-data-001
[junit4] 2> 524452 INFO (SUITE-CustomCollectionTest-seed#[A403300B1438A1D2]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 524452 INFO (SUITE-CustomCollectionTest-seed#[A403300B1438A1D2]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 524453 INFO (SUITE-CustomCollectionTest-seed#[A403300B1438A1D2]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.3/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CustomCollectionTest_A403300B1438A1D2-001/tempDir-001
[junit4] 2> 524453 INFO (SUITE-CustomCollectionTest-seed#[A403300B1438A1D2]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 524468 INFO (Thread-309) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 524468 INFO (Thread-309) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 524469 ERROR (Thread-309) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 524577 INFO (SUITE-CustomCollectionTest-seed#[A403300B1438A1D2]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:37306
[junit4] 2> 524724 INFO (zkConnectionManagerCallback-434-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 524742 INFO (jetty-launcher-431-thread-1) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T23:27:37+02:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 524761 INFO (jetty-launcher-431-thread-2) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T23:27:37+02:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 524761 INFO (jetty-launcher-431-thread-3) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T23:27:37+02:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 524927 INFO (jetty-launcher-431-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 524927 INFO (jetty-launcher-431-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 524927 INFO (jetty-launcher-431-thread-2) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 524928 INFO (jetty-launcher-431-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@60ecbccd{/solr,null,AVAILABLE}
[junit4] 2> 524929 INFO (jetty-launcher-431-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@25f66306{HTTP/1.1,[http/1.1]}{127.0.0.1:46432}
[junit4] 2> 524929 INFO (jetty-launcher-431-thread-2) [ ] o.e.j.s.Server Started @538446ms
[junit4] 2> 524929 INFO (jetty-launcher-431-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46432}
[junit4] 2> 524930 ERROR (jetty-launcher-431-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 524930 INFO (jetty-launcher-431-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.1
[junit4] 2> 524930 INFO (jetty-launcher-431-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 524930 INFO (jetty-launcher-431-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 524930 INFO (jetty-launcher-431-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-05-11T15:51:56.267Z
[junit4] 2> 524965 INFO (jetty-launcher-431-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 524965 INFO (jetty-launcher-431-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 524965 INFO (jetty-launcher-431-thread-1) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 525016 INFO (jetty-launcher-431-thread-3) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 525016 INFO (jetty-launcher-431-thread-3) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 525016 INFO (jetty-launcher-431-thread-3) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 525017 INFO (jetty-launcher-431-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@74ea8427{/solr,null,AVAILABLE}
[junit4] 2> 525017 INFO (jetty-launcher-431-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@61748f20{HTTP/1.1,[http/1.1]}{127.0.0.1:37731}
[junit4] 2> 525017 INFO (jetty-launcher-431-thread-3) [ ] o.e.j.s.Server Started @538534ms
[junit4] 2> 525017 INFO (jetty-launcher-431-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37731}
[junit4] 2> 525018 ERROR (jetty-launcher-431-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 525018 INFO (jetty-launcher-431-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.1
[junit4] 2> 525018 INFO (jetty-launcher-431-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 525018 INFO (jetty-launcher-431-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 525018 INFO (jetty-launcher-431-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-05-11T15:51:56.355Z
[junit4] 2> 525098 INFO (jetty-launcher-431-thread-4) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T23:27:37+02:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 525130 INFO (zkConnectionManagerCallback-438-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 525134 INFO (jetty-launcher-431-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 525146 INFO (jetty-launcher-431-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@455792f1{/solr,null,AVAILABLE}
[junit4] 2> 525147 INFO (jetty-launcher-431-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5fe40350{HTTP/1.1,[http/1.1]}{127.0.0.1:38350}
[junit4] 2> 525147 INFO (jetty-launcher-431-thread-1) [ ] o.e.j.s.Server Started @538664ms
[junit4] 2> 525147 INFO (jetty-launcher-431-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38350}
[junit4] 2> 525147 ERROR (jetty-launcher-431-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 525147 INFO (jetty-launcher-431-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.1
[junit4] 2> 525147 INFO (jetty-launcher-431-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 525147 INFO (jetty-launcher-431-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 525147 INFO (jetty-launcher-431-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-05-11T15:51:56.484Z
[junit4] 2> 525166 INFO (zkConnectionManagerCallback-436-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 525167 INFO (jetty-launcher-431-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 525264 INFO (zkConnectionManagerCallback-440-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 525265 INFO (jetty-launcher-431-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 525302 INFO (jetty-launcher-431-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37306/solr
[junit4] 2> 525373 INFO (jetty-launcher-431-thread-4) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 525411 INFO (jetty-launcher-431-thread-4) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 525412 INFO (jetty-launcher-431-thread-4) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 525463 INFO (jetty-launcher-431-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37306/solr
[junit4] 2> 525464 INFO (zkConnectionManagerCallback-444-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 525629 INFO (zkConnectionManagerCallback-448-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 525632 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x101f42955070005, likely client has closed socket
[junit4] 2> 525517 INFO (jetty-launcher-431-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5ed7d8f0{/solr,null,AVAILABLE}
[junit4] 2> 525642 INFO (jetty-launcher-431-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37306/solr
[junit4] 2> 525644 INFO (jetty-launcher-431-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7402f403{HTTP/1.1,[http/1.1]}{127.0.0.1:40231}
[junit4] 2> 525683 INFO (jetty-launcher-431-thread-4) [ ] o.e.j.s.Server Started @539200ms
[junit4] 2> 525683 INFO (jetty-launcher-431-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40231}
[junit4] 2> 525683 ERROR (jetty-launcher-431-thread-4) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 525790 INFO (jetty-launcher-431-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.1
[junit4] 2> 525790 INFO (jetty-launcher-431-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 525790 INFO (jetty-launcher-431-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 525791 INFO (jetty-launcher-431-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-05-11T15:51:57.128Z
[junit4] 2> 525678 INFO (zkConnectionManagerCallback-454-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 525814 INFO (zkConnectionManagerCallback-452-thread-1-processing-n:127.0.0.1:37731_solr) [n:127.0.0.1:37731_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 525865 INFO (zkConnectionManagerCallback-458-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 525885 INFO (jetty-launcher-431-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 525885 INFO (zkConnectionManagerCallback-456-thread-1-processing-n:127.0.0.1:46432_solr) [n:127.0.0.1:46432_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 525951 INFO (zkConnectionManagerCallback-461-thread-1-processing-n:127.0.0.1:38350_solr) [n:127.0.0.1:38350_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 526085 INFO (jetty-launcher-431-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37306/solr
[junit4] 2> 526227 INFO (zkConnectionManagerCallback-465-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 526377 INFO (zkConnectionManagerCallback-467-thread-1-processing-n:127.0.0.1:40231_solr) [n:127.0.0.1:40231_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 526719 INFO (jetty-launcher-431-thread-1) [n:127.0.0.1:38350_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 526720 INFO (jetty-launcher-431-thread-2) [n:127.0.0.1:46432_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 526720 INFO (jetty-launcher-431-thread-3) [n:127.0.0.1:37731_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 526721 INFO (jetty-launcher-431-thread-1) [n:127.0.0.1:38350_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38350_solr
[junit4] 2> 526811 INFO (jetty-launcher-431-thread-3) [n:127.0.0.1:37731_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37731_solr
[junit4] 2> 526811 INFO (jetty-launcher-431-thread-2) [n:127.0.0.1:46432_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46432_solr
[junit4] 2> 526828 INFO (jetty-launcher-431-thread-1) [n:127.0.0.1:38350_solr ] o.a.s.c.Overseer Overseer (id=72607527371997194-127.0.0.1:38350_solr-n_0000000000) starting
[junit4] 2> 527128 INFO (zkCallback-451-thread-1-processing-n:127.0.0.1:37731_solr) [n:127.0.0.1:37731_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 527128 INFO (zkCallback-460-thread-1-processing-n:127.0.0.1:38350_solr) [n:127.0.0.1:38350_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 527239 INFO (zkCallback-455-thread-1-processing-n:127.0.0.1:46432_solr) [n:127.0.0.1:46432_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 527664 INFO (zkConnectionManagerCallback-474-thread-1-processing-n:127.0.0.1:38350_solr) [n:127.0.0.1:38350_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 527686 INFO (jetty-launcher-431-thread-1) [n:127.0.0.1:38350_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 527687 INFO (jetty-launcher-431-thread-1) [n:127.0.0.1:38350_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37306/solr ready
[junit4] 2> 528119 INFO (jetty-launcher-431-thread-1) [n:127.0.0.1:38350_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38350_solr
[junit4] 2> 528163 INFO (jetty-launcher-431-thread-3) [n:127.0.0.1:37731_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37731.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41b221f6
[junit4] 2> 528202 INFO (zkCallback-460-thread-1-processing-n:127.0.0.1:38350_solr) [n:127.0.0.1:38350_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 528202 INFO (zkCallback-451-thread-1-processing-n:127.0.0.1:37731_solr) [n:127.0.0.1:37731_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 528255 INFO (jetty-launcher-431-thread-2) [n:127.0.0.1:46432_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46432.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41b221f6
[junit4] 2> 528301 INFO (zkCallback-455-thread-1-processing-n:127.0.0.1:46432_solr) [n:127.0.0.1:46432_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 528435 INFO (zkCallback-473-thread-1-processing-n:127.0.0.1:38350_solr) [n:127.0.0.1:38350_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 528496 INFO (jetty-launcher-431-thread-2) [n:127.0.0.1:46432_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46432.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41b221f6
[junit4] 2> 528602 INFO (jetty-launcher-431-thread-2) [n:127.0.0.1:46432_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46432.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41b221f6
[junit4] 2> 528604 INFO (jetty-launcher-431-thread-2) [n:127.0.0.1:46432_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.3/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CustomCollectionTest_A403300B1438A1D2-001/tempDir-001/node2/.
[junit4] 2> 528718 INFO (jetty-launcher-431-thread-3) [n:127.0.0.1:37731_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37731.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41b221f6
[junit4] 2> 528718 INFO (jetty-launcher-431-thread-3) [n:127.0.0.1:37731_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37731.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41b221f6
[junit4] 2> 528741 INFO (zkConnectionManagerCallback-478-thread-1-processing-n:127.0.0.1:46432_solr) [n:127.0.0.1:46432_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 528749 INFO (jetty-launcher-431-thread-3) [n:127.0.0.1:37731_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.3/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CustomCollectionTest_A403300B1438A1D2-001/tempDir-001/node3/.
[junit4] 2> 528850 INFO (jetty-launcher-431-thread-2) [n:127.0.0.1:46432_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 528923 INFO (jetty-launcher-431-thread-2) [n:127.0.0.1:46432_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37306/solr ready
[junit4] 2> 529072 INFO (zkConnectionManagerCallback-482-thread-1-processing-n:127.0.0.1:37731_solr) [n:127.0.0.1:37731_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 529298 INFO (jetty-launcher-431-thread-1) [n:127.0.0.1:38350_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38350.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41b221f6
[junit4] 2> 529312 INFO (jetty-launcher-431-thread-3) [n:127.0.0.1:37731_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 529383 INFO (jetty-launcher-431-thread-1) [n:127.0.0.1:38350_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38350.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41b221f6
[junit4] 2> 529383 INFO (jetty-launcher-431-thread-1) [n:127.0.0.1:38350_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38350.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41b221f6
[junit4] 2> 529384 INFO (jetty-launcher-431-thread-1) [n:127.0.0.1:38350_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.3/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CustomCollectionTest_A403300B1438A1D2-001/tempDir-001/node1/.
[junit4] 2> 529426 INFO (jetty-launcher-431-thread-3) [n:127.0.0.1:37731_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37306/solr ready
[junit4] 2> 529801 INFO (jetty-launcher-431-thread-4) [n:127.0.0.1:40231_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 529836 INFO (jetty-launcher-431-thread-4) [n:127.0.0.1:40231_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 529838 INFO (jetty-launcher-431-thread-4) [n:127.0.0.1:40231_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40231_solr
[junit4] 2> 529852 INFO (zkCallback-455-thread-1-processing-n:127.0.0.1:46432_solr) [n:127.0.0.1:46432_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 529852 INFO (zkCallback-451-thread-1-processing-n:127.0.0.1:37731_solr) [n:127.0.0.1:37731_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 529856 INFO (zkCallback-473-thread-1-processing-n:127.0.0.1:38350_solr) [n:127.0.0.1:38350_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 530022 INFO (zkCallback-477-thread-1-processing-n:127.0.0.1:46432_solr) [n:127.0.0.1:46432_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 530022 INFO (zkCallback-466-thread-1-processing-n:127.0.0.1:40231_solr) [n:127.0.0.1:40231_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 530022 INFO (zkCallback-481-thread-1-processing-n:127.0.0.1:37731_solr) [n:127.0.0.1:37731_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 530079 INFO (zkCallback-460-thread-2-processing-n:127.0.0.1:38350_solr) [n:127.0.0.1:38350_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 530399 INFO (jetty-launcher-431-thread-4) [n:127.0.0.1:40231_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40231.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41b221f6
[junit4] 2> 530508 INFO (jetty-launcher-431-thread-4) [n:127.0.0.1:40231_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40231.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41b221f6
[junit4] 2> 530526 INFO (jetty-launcher-431-thread-4) [n:127.0.0.1:40231_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40231.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41b221f6
[junit4] 2> 530527 INFO (jetty-launcher-431-thread-4) [n:127.0.0.1:40231_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.3/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CustomCollectionTest_A403300B1438A1D2-001/tempDir-001/node4/.
[junit4] 2> 530637 INFO (zkConnectionManagerCallback-486-thread-1-processing-n:127.0.0.1:40231_solr) [n:127.0.0.1:40231_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 530638 INFO (jetty-launcher-431-thread-4) [n:127.0.0.1:40231_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 530712 INFO (jetty-launcher-431-thread-4) [n:127.0.0.1:40231_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37306/solr ready
[junit4] 2> 530984 INFO (zkConnectionManagerCallback-488-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 531090 INFO (zkConnectionManagerCallback-492-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 531091 INFO (SUITE-CustomCollectionTest-seed#[A403300B1438A1D2]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 531110 INFO (SUITE-CustomCollectionTest-seed#[A403300B1438A1D2]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37306/solr ready
[junit4] 2> 531347 INFO (TEST-CustomCollectionTest.testRouteFieldForImplicitRouter-seed#[A403300B1438A1D2]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testRouteFieldForImplicitRouter
[junit4] 2> 531348 INFO (TEST-CustomCollectionTest.testRouteFieldForImplicitRouter-seed#[A403300B1438A1D2]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 531493 INFO (qtp1304913295-1392) [n:127.0.0.1:46432_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params shards=a,b,c,d&replicationFactor=3&collection.configName=conf&maxShardsPerNode=4&router.field=shard_s&name=withShardField&router.name=implicit&nrtReplicas=3&action=CREATE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 531512 INFO (OverseerThreadFactory-465-thread-1-processing-n:127.0.0.1:38350_solr) [n:127.0.0.1:38350_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection withShardField
[junit4] 2> 531678 INFO (OverseerStateUpdate-72607527371997194-127.0.0.1:38350_solr-n_0000000000) [n:127.0.0.1:38350_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"withShardField",
[junit4] 2> "shard":"a",
[junit4] 2> "core":"withShardField_a_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:38350/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 531820 INFO (OverseerStateUpdate-72607527371997194-127.0.0.1:38350_solr-n_0000000000) [n:127.0.0.1:38350_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"withShardField",
[junit4] 2> "shard":"a",
[junit4] 2> "core":"withShardField_a_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:46432/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 531858 INFO (OverseerStateUpdate-72607527371997194-127.0.0.1:38350_solr-n_0000000000) [n:127.0.0.1:38350_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"withShardField",
[junit4] 2> "shard":"a",
[junit4] 2> "core":"withShardField_a_replica_n3",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:37731/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 531859 INFO (OverseerStateUpdate-72607527371997194-127.0.0.1:38350_solr-n_0000000000) [n:127.0.0.1:38350_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"withShardField",
[junit4] 2> "shard":"b",
[junit4] 2> "core":"withShardField_b_replica_n4",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:40231/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 531860 INFO (OverseerStateUpdate-72607527371997194-127.0.0.1:38350_solr-n_0000000000) [n:127.0.0.1:38350_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"withShardField",
[junit4] 2> "shard":"b",
[junit4] 2> "core":"withShardField_b_replica_n5",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:38350/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 531931 INFO (OverseerStateUpdate-72607527371997194-127.0.0.1:38350_solr-n_0000000000) [n:127.0.0.1:38350_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"withShardField",
[junit4] 2> "shard":"b",
[junit4] 2> "core":"withShardField_b_replica_n6",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:46432/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 531934 INFO (OverseerStateUpdate-72607527371997194-127.0.0.1:38350_solr-n_0000000000) [n:127.0.0.1:38350_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"withShardField",
[junit4] 2> "shard":"c",
[junit4] 2> "core":"withShardField_c_replica_n8",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:37731/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 532004 INFO (OverseerStateUpdate-72607527371997194-127.0.0.1:38350_solr-n_0000000000) [n:127.0.0.1:38350_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"withShardField",
[junit4] 2> "shard":"c",
[junit4] 2> "core":"withShardField_c_replica_n9",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:40231/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 532005 INFO (OverseerStateUpdate-72607527371997194-127.0.0.1:38350_solr-n_0000000000) [n:127.0.0.1:38350_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"withShardField",
[junit4] 2> "shard":"c",
[junit4] 2> "core":"withShardField_c_replica_n10",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:38350/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 532006 INFO (OverseerStateUpdate-72607527371997194-127.0.0.1:38350_solr-n_0000000000) [n:127.0.0.1:38350_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"withShardField",
[junit4] 2> "shard":"d",
[junit4] 2> "core":"withShardField_d_replica_n11",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:46432/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 532007 INFO (OverseerStateUpdate-72607527371997194-127.0.0.1:38350_solr-n_0000000000) [n:127.0.0.1:38350_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"withShardField",
[junit4] 2> "shard":"d",
[junit4] 2> "core":"withShardField_d_replica_n15",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:37731/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 532219 INFO (OverseerStateUpdate-72607527371997194-127.0.0.1:38350_solr-n_0000000000) [n:127.0.0.1:38350_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"withShardField",
[junit4] 2> "shard":"d",
[junit4] 2> "core":"withShardField_d_replica_n16",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:40231/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 532790 INFO (qtp1304913295-1393) [n:127.0.0.1:46432_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node18&collection.configName=conf&newCollection=true&name=withShardField_b_replica_n6&action=CREATE&numShards=4&collection=withShardField&shard=b&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 532791 INFO (qtp1304913295-1393) [n:127.0.0.1:46432_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 532808 INFO (qtp1503149944-1390) [n:127.0.0.1:37731_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node13&collection.configName=conf&newCollection=true&name=withShardField_a_replica_n3&action=CREATE&numShards=4&collection=withShardField&shard=a&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 532809 INFO (qtp1503149944-1390) [n:127.0.0.1:37731_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 532934 INFO (qtp1503149944-1403) [n:127.0.0.1:37731_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node23&collection.configName=conf&newCollection=true&name=withShardField_d_replica_n15&action=CREATE&numShards=4&collection=withShardField&shard=d&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 532951 INFO (qtp1304913295-1395) [n:127.0.0.1:46432_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node12&collection.configName=conf&newCollection=true&name=withShardField_a_replica_n2&action=CREATE&numShards=4&collection=withShardField&shard=a&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 532952 INFO (qtp1503149944-1406) [n:127.0.0.1:37731_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node19&collection.configName=conf&newCollection=true&name=withShardField_c_replica_n8&action=CREATE&numShards=4&collection=withShardField&shard=c&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 532953 INFO (qtp1304913295-1397) [n:127.0.0.1:46432_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node22&collection.configName=conf&newCollection=true&name=withShardField_d_replica_n11&action=CREATE&numShards=4&collection=withShardField&shard=d&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 533097 INFO (qtp1270670927-1414) [n:127.0.0.1:40231_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node14&collection.configName=conf&newCollection=true&name=withShardField_b_replica_n4&action=CREATE&numShards=4&collection=withShardField&shard=b&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 533137 INFO (qtp1270670927-1420) [n:127.0.0.1:40231_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node20&collection.configName=conf&newCollection=true&name=withShardField_c_replica_n9&action=CREATE&numShards=4&collection=withShardField&shard=c&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 533138 INFO (qtp1270670927-1420) [n:127.0.0.1:40231_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 533138 INFO (qtp1270670927-1428) [n:127.0.0.1:40231_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node24&collection.configName=conf&newCollection=true&name=withShardField_d_replica_n16&action=CREATE&numShards=4&collection=withShardField&shard=d&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 533190 INFO (qtp1933868709-1385) [n:127.0.0.1:38350_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=withShardField_a_replica_n1&action=CREATE&numShards=4&collection=withShardField&shard=a&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 533191 INFO (qtp1933868709-1385) [n:127.0.0.1:38350_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 533191 INFO (qtp1933868709-1383) [n:127.0.0.1:38350_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node21&collection.configName=conf&newCollection=true&name=withShardField_c_replica_n10&action=CREATE&numShards=4&collection=withShardField&shard=c&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 533412 INFO (qtp1933868709-1401) [n:127.0.0.1:38350_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node17&collection.configName=conf&newCollection=true&name=withShardField_b_replica_n5&action=CREATE&numShards=4&collection=withShardField&shard=b&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 533603 INFO (zkCallback-466-thread-1-processing-n:127.0.0.1:40231_solr) [n:127.0.0.1:40231_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/withShardField/state.json] for collection [withShardField] has occurred - updating... (live nodes size: [4])
[junit4] 2> 533603 INFO (zkCallback-455-thread-1-processing-n:127.0.0.1:46432_solr) [n:127.0.0.1:46432_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/withShardField/state.json] for collection [withShardField] has occurred - updating... (live nodes size: [4])
[junit4] 2> 533607 INFO (zkCallback-466-thread-2-processing-n:127.0.0.1:40231_solr) [n:127.0.0.1:40231_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/withShardField/state.json] for collection [withShardField] has occurred - updating... (live nodes size: [4])
[junit4] 2> 533616 INFO (zkCallback-460-thread-2-processing-n:127.0.0.1:38350_solr) [n:127.0.0.1:38350_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/withShardField/state.json] for collection [withShardField] has occurred - updating... (live nodes size: [4])
[junit4] 2> 533624 INFO (zkCallback-455-thread-2-processing-n:127.0.0.1:46432_solr) [n:127.0.0.1:46432_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/withShardField/state.json] for collection [withShardField] has occurred - updating... (live nodes size: [4])
[junit4] 2> 533624 INFO (zkCallback-455-thread-3-processing-n:127.0.0.1:46432_solr) [n:127.0.0.1:46432_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/withShardField/state.json] for collection [withShardField] has occurred - updating... (live nodes size: [4])
[junit4] 2> 533624 INFO (zkCallback-455-thread-4-processing-n:127.0.0.1:46432_solr) [n:127.0.0.1:46432_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/withShardField/state.json] for collection [withShardField] has occurred - updating... (live nodes size: [4])
[junit4] 2> 533644 INFO (zkCallback-451-thread-1-processing-n:127.0.0.1:37731_solr) [n:127.0.0.1:37731_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/withShardField/state.json] for collection [withShardField] has occurred - updating... (live nodes size: [4])
[junit4] 2> 534053 INFO (qtp1503149944-1406) [n:127.0.0.1:37731_solr c:withShardField s:c r:core_node19 x:withShardField_c_replica_n8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.1
[junit4] 2> 534073 INFO (qtp1503149944-1403) [n:127.0.0.1:37731_solr c:withShardField s:d r:core_node23 x:withShardField_d_replica_n15] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.1
[junit4] 2> 539597 WARN (jetty-launcher-431-thread-1-SendThread(127.0.0.1:37306)) [n:127.0.0.1:38350_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 7321ms for sessionid 0x101f4295507000d
[junit4] 2> 539710 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x101f4295507000d, likely client has closed socket
[junit4] 2> 539701 INFO (qtp1503149944-1403) [n:127.0.0.1:37731_solr c:withShardField s:d r:core_node23 x:withShardField_d_replica_n15] o.a.s.s.IndexSchema [withShardField_d_replica_n15] Schema name=cloud-dynamic
[junit4] 2> 539601 WARN (jetty-launcher-431-thread-2-SendThread(127.0.0.1:37306)) [n:127.0.0.1:46432_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 8055ms for sessionid 0x101f4295507000e
[junit4] 2> 539601 WARN (SUITE-CustomCollectionTest-seed#[A403300B1438A1D2]-worker-SendThread(127.0.0.1:37306)) [ ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 8036ms for sessionid 0x101f42955070012
[junit4] 2> 539601 WARN (jetty-launcher-431-thread-3-SendThread(127.0.0.1:37306)) [n:127.0.0.1:37731_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 8035ms for sessionid 0x101f4295507000f
[junit4] 2> 539601 WARN (jetty-launcher-431-thread-4-SendThread(127.0.0.1:37306)) [n:127.0.0.1:40231_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 7996ms for sessionid 0x101f42955070010
[junit4] 2> 539829 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x101f4295507000e, likely client has closed socket
[junit4] 2> 539830 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x101f4295507000f, likely client has closed socket
[junit4] 2> 539830 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x101f42955070010, likely client has closed socket
[junit4] 2> 539830 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x101f42955070012, likely client has closed socket
[junit4] 2> 539818 WARN (zkConnectionManagerCallback-474-thread-1-processing-n:127.0.0.1:38350_solr) [n:127.0.0.1:38350_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4982f00b name: ZooKeeperConnection Watcher:127.0.0.1:37306/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 539836 WARN (zkConnectionManagerCallback-474-thread-1-processing-n:127.0.0.1:38350_solr) [n:127.0.0.1:38350_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 539817 INFO (qtp1503149944-1406) [n:127.0.0.1:37731_solr c:withShardField s:c r:core_node19 x:withShardField_c_replica_n8] o.a.s.s.IndexSchema [withShardField_c_replica_n8] Schema name=cloud-dynamic
[junit4] 2> 539793 INFO (qtp1503149944-1390) [n:127.0.0.1:37731_solr c:withShardField s:a r:core_node13 x:withShardField_a_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.1
[junit4] 2> 540019 WARN (zkConnectionManagerCallback-492-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@61b5667e name: ZooKeeperConnection Watcher:127.0.0.1:37306/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 540019 WARN (zkConnectionManagerCallback-492-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 539966 INFO (qtp1503149944-1403) [n:127.0.0.1:37731_solr c:withShardField s:d r:core_node23 x:withShardField_d_replica_n15] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 540022 INFO (qtp1503149944-1403) [n:127.0.0.1:37731_solr c:withShardField s:d r:core_node23 x:withShardField_d_replica_n15] o.a.s.c.CoreContainer Creating SolrCore 'withShardField_d_replica_n15' using configuration from collection withShardField, trusted=true
[junit4] 2> 540022 INFO (qtp1503149944-1403) [n:127.0.0.1:37731_solr c:withShardField s:d r:core_node23 x:withShardField_d_replica_n15] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37731.solr.core.withShardField.d.replica_n15' (registry 'solr.core.withShardField.d.replica_n15') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41b221f6
[junit4] 2> 540022 INFO (qtp1503149944-1403) [n:127.0.0.1:37731_solr c:withShardField s:d r:core_node23 x:withShardField_d_replica_n15] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 540023 INFO (qtp1503149944-1403) [n:127.0.0.1:37731_solr c:withShardField s:d r:core_node23 x:withShardField_d_replica_n15] o.a.s.c.SolrCore [[withShardField_d_replica_n15] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.3/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CustomCollectionTest_A403300B1438A1D2-001/tempDir-001/node3/withShardField_d_replica_n15], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.3/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CustomCollectionTest_A403300B1438A1D2-001/tempDir-001/node3/./withShardField_d_replica_n15/data/]
[junit4] 2> 540047 WARN (zkConnectionManagerCallback-478-thread-1-processing-n:127.0.0.1:46432_solr) [n:127.0.0.1:46432_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4d67b4c4 name: ZooKeeperConnection Watcher:127.0.0.1:37306/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 540047 WARN (zkConnectionManagerCallback-478-thread-1-processing-n:127.0.0.1:46432_solr) [n:127.0.0.1:46432_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 540108 WARN (zkConnectionManagerCallback-486-thread-1-processing-n:127.0.0.1:40231_solr) [n:127.0.0.1:40231_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@70d386ba name: ZooKeeperConnection Watcher:127.0.0.1:37306/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 540108 WARN (zkConnectionManagerCallback-486-thread-1-processing-n:127.0.0.1:40231_solr) [n:127.0.0.1:40231_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 540120 WARN (zkConnectionManagerCallback-482-thread-1-processing-n:127.0.0.1:37731_solr) [n:127.0.0.1:37731_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@657c00cf name: ZooKeeperConnection Watcher:127.0.0.1:37306/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 540120 WARN (zkConnectionManagerCallback-482-thread-1-processing-n:127.0.0.1:37731_solr) [n:127.0.0.1:37731_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 540236 INFO (qtp1304913295-1397) [n:127.0.0.1:46432_solr c:withShardField s:d r:core_node22 x:withShardField_d_replica_n11] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.1
[junit4] 2> 540273 INFO (qtp1503149944-1390) [n:127.0.0.1:37731_solr c:withShardField s:a r:core_node13 x:withShardField_a_replica_n3] o.a.s.s.IndexSchema [withShardField_a_replica_n3] Schema name=cloud-dynamic
[junit4] 2> 541524 INFO (qtp1304913295-1397) [n:127.0.0.1:46432_solr c:withShardField s:d r:core_node22 x:withShardField_d_replica_n11] o.a.s.s.IndexSchema [withShardField_d_replica_n11] Schema name=cloud-dynamic
[junit4] 2> 541610 INFO (zkConnectionManagerCallback-478-thread-1-processing-n:127.0.0.1:46432_solr) [n:127.0.0.1:46432_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 541612 INFO (qtp1503149944-1406) [n:127.0.0.1:37731_solr c:withShardField s:c r:core_node19 x:withShardField_c_replica_n8] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 541612 INFO (qtp1503149944-1406) [n:127.0.0.1:37731_solr c:withShardField s:c r:core_node19 x:withShardField_c_replica_n8] o.a.s.c.CoreContainer Creating SolrCore 'withShardField_c_replica_n8' using configuration from collection withShardField, trusted=true
[junit4] 2> 541612 INFO (qtp1270670927-1414) [n:127.0.0.1:40231_solr c:withShardField s:b r:core_node14 x:withShardField_b_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.1
[junit4] 2> 541612 INFO (qtp1503149944-1406) [n:127.0.0.1:37731_solr c:withShardField s:c r:core_node19 x:withShardField_c_replica_n8] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37731.solr.core.withShardField.c.replica_n8' (registry 'solr.core.withShardField.c.replica_n8') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41b221f6
[junit4] 2> 541612 INFO (qtp1503149944-1406) [n:127.0.0.1:37731_solr c:withShardField s:c r:core_node19 x:withShardField_c_replica_n8] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 541613 INFO (qtp1503149944-1406) [n:127.0.0.1:37731_solr c:withShardField s:c r:core_node19 x:withShardField_c_replica_n8] o.a.s.c.SolrCore [[withShardField_c_replica_n8] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.3/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CustomCollectionTest_A403300B1438A1D2-001/tempDir-001/node3/withShardField_c_replica_n8], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.3/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CustomCollectionTest_A403300B1438A1D2-001/tempDir-001/node3/./withShardField_c_replica_n8/data/]
[junit4] 2> 541628 INFO (qtp1304913295-1395) [n:127.0.0.1:46432_solr c:withShardField s:a r:core_node12 x:withShardField_a_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.1
[junit4] 2> 541632 INFO (qtp1933868709-1385) [n:127.0.0.1:38350_solr c:withShardField s:a r:core_node7 x:withShardField_a_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.1
[junit4] 2> 541993 INFO (qtp1933868709-1383) [n:127.0.0.1:38350_solr c:withShardField s:c r:core_node21 x:withShardField_c_replica_n10] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.1
[junit4] 2> 541995 INFO (qtp1304913295-1393) [n:127.0.0.1:46432_solr c:withShardField s:b r:core_node18 x:withShardField_b_replica_n6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.1
[junit4] 2> 542009 INFO (qtp1270670927-1428) [n:127.0.0.1:40231_solr c:withShardField s:d r:core_node24 x:withShardField_d_replica_n16] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.1
[junit4] 2> 542014 INFO (qtp1304913295-1397) [n:127.0.0.1:46432_solr c:withShardField s:d r:core_node22 x:withShardField_d_replica_n11] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 542014 INFO (qtp1304913295-1397) [n:127.0.0.1:46432_solr c:withShardField s:d r:core_node22 x:withShardField_d_replica_n11] o.a.s.c.CoreContainer Creating SolrCore 'withShardField_d_replica_n11' using configuration from collection withShardField, trusted=true
[junit4] 2> 542015 INFO (qtp1304913295-1397) [n:127.0.0.1:46432_solr c:withShardField s:d r:core_node22 x:withShardField_d_replica_n11] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46432.solr.core.withShardField.d.replica_n11' (registry 'solr.core.withShardField.d.replica_n11') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@41b221f6
[junit4] 2> 542015 INFO (qtp1304913295-1397) [n:127.0.0.1:46432_solr c:withShardField s:d r:core_node22 x:withShardField_d_replica_n11] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 542015 INFO (qtp1304913295-1397) [n:127.0.0.1:46432_solr c:withShardField s:d r:core_node22 x:withShardField_d_replica_n11] o.a.s.c.SolrCore [[withShardField_d_replica_n11] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.3/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CustomCollectionTest_A403300B1438A1D2-001/tempDir-001/node2/withShardField_d_replica_n11], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.3/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CustomCollectionTest_A403300B1438A1D2-001/tempDir-001/node2/./withShardField_d_replica_n11/data/]
[junit4] 2> 542029 INFO (qtp1933868709-1401) [n:127.0.0.1:38350_solr c:withShardField s:b r:core_node17 x:withShardField_b_replica_n5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.1
[junit4] 2> 542029 INFO (qtp1270670927-1420) [n:127.0.0.1:40231_solr c:withShardField s:c r:core_node20 x:withShardField_c_replica_n9] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.1
[junit4] 2> 542287 INFO (qtp1304913295-1393) [n:127.0.0.1:46432_solr c:withShardField s:b r:core_node18 x:withShardField_b_replica_n6] o.a.s.s.IndexSchema [withShardField_b_replica_n6] Schema name=cloud-dynamic
[junit4] 2> 542343 INFO (qtp1933868709-1385) [n:127.0.0.1:38350_solr c:withShardField s:a r:core_node7 x:withShardField_a_replica_n1] o.a.s.s.IndexSchema [withShardField_a_replica_n1] Schema name=cloud-dynamic
[junit4] 2> 542375 INFO (qtp1933868709-1383) [n:127.0.0.1:38350_solr c:withShardField s:c r:core_node21 x:withShardField_c_replica_n10] o.a.s.s.IndexSchema [withShardField_c_replica_n10] Schema name=cloud-dynamic
[junit4] 2> 542382 INFO (qtp1304913295-1395) [n:127.0.0.1:46432_solr c:withShardField s:a r:core_node12 x:withShardField_a_replica_n2] o.a.s.s.IndexSchema [withShardField_a_replica_n2] Schema name=cloud-dynamic
[junit4] 2> 542394 INFO (qtp1270670927-1414) [n:127.0.0.1:40231_solr c:withShardField s:b r:core_node14 x:withShardField_b_replica_n4] o.a.s.s.IndexSchema [withShardField_b_replica_n4] Schema name=cloud-dynamic
[junit4] 2> 542395 INFO (qtp1270670927-1428) [n:127.0.0.1:40231_solr c:withShardField s:d r:core_node24 x:withShardField_d_replica_n16] o.a.s.s.IndexSchema [withShardField_d_replica_n16] Schema name=cloud-dynamic
[junit4] 2> 542438 INFO (qtp1270670927-1420) [n:127.0.0.1:40231_solr c:withShardField s:c r:core_node20 x:withShardField_c_replica_n9] o.a.s.s.IndexSchema [withShardField_c_replica_n9] Schema name=cloud-dynamic
[junit4] 2> 542514 INFO (qtp1933868709-1401) [n:127.0.0.1:38350_solr c:withShardField s:b r:core_node17 x:withShardField_b_replica_n5] o.a.s.s.IndexSchema [withShardField_b_replica_n5] Schema name=cloud-dynamic
[junit4] 2> 542532 WARN (SUITE-CustomCollectionTest-seed#[A403300B1438A1D2]-worker-SendThread(127.0.0.1:37306)) [ ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x101f42955070012 has expired
[junit4] 2> 542532 WARN (jetty-launcher-431-threa
[...truncated too long message...]
change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 327266 INFO (zkCallback-92-thread-3-processing-n:127.0.0.1:34302_solr) [n:127.0.0.1:34302_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 327268 INFO (zkCallback-94-thread-6-processing-n:127.0.0.1:40318_solr) [n:127.0.0.1:40318_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 327269 INFO (zkCallback-100-thread-2-processing-n:127.0.0.1:34302_solr) [n:127.0.0.1:34302_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 327272 INFO (zkCallback-104-thread-5-processing-n:127.0.0.1:40318_solr) [n:127.0.0.1:40318_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 327285 INFO (zkCallback-94-thread-2-processing-n:127.0.0.1:40318_solr) [n:127.0.0.1:40318_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 327285 INFO (zkCallback-94-thread-1-processing-n:127.0.0.1:40318_solr) [n:127.0.0.1:40318_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 327396 INFO (coreCloseExecutor-473-thread-1) [n:127.0.0.1:34302_solr c:collection1_collection s:shard1 r:core_node3 x:collection1_collection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1_collection.shard1.leader, tag=83648577
[junit4] 2> 327592 INFO (coreCloseExecutor-474-thread-1) [n:127.0.0.1:40318_solr c:collection1_collection s:shard2 r:core_node4 x:collection1_collection_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1_collection.shard2.leader, tag=93050922
[junit4] 2> 327772 INFO (zkCallback-94-thread-2-processing-n:127.0.0.1:40318_solr) [n:127.0.0.1:40318_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 327772 INFO (zkCallback-94-thread-6-processing-n:127.0.0.1:40318_solr) [n:127.0.0.1:40318_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 327847 INFO (zkCallback-92-thread-2-processing-n:127.0.0.1:34302_solr) [n:127.0.0.1:34302_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 327847 INFO (zkCallback-92-thread-4-processing-n:127.0.0.1:34302_solr) [n:127.0.0.1:34302_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 327990 INFO (jetty-closer-77-thread-2) [ ] o.a.s.c.Overseer Overseer (id=72607922758025221-127.0.0.1:34302_solr-n_0000000000) closing
[junit4] 2> 328028 INFO (OverseerStateUpdate-72607922758025221-127.0.0.1:34302_solr-n_0000000000) [n:127.0.0.1:34302_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:34302_solr
[junit4] 2> 328029 WARN (OverseerAutoScalingTriggerThread-72607922758025221-127.0.0.1:34302_solr-n_0000000000) [n:127.0.0.1:34302_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 328100 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x101f48563df0006, likely client has closed socket
[junit4] 2> 328118 INFO (jetty-closer-77-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2392b698{/solr,null,UNAVAILABLE}
[junit4] 2> 328119 INFO (jetty-closer-77-thread-1) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 328239 INFO (jetty-closer-77-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@42375d27{/solr,null,UNAVAILABLE}
[junit4] 2> 328240 INFO (jetty-closer-77-thread-2) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 328280 ERROR (SUITE-JdbcTest-seed#[AA2728CD153CBC80]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 328282 INFO (SUITE-JdbcTest-seed#[AA2728CD153CBC80]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38629 38629
[junit4] 2> 328577 INFO (Thread-370) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38629 38629
[junit4] 2> 328579 WARN (Thread-370) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 9 /solr/aliases.json
[junit4] 2> 2 /solr/security.json
[junit4] 2> 2 /solr/configs/conf
[junit4] 2> 2 /solr/collections/collection1_collection/terms/shard1
[junit4] 2> 2 /solr/collections/collection1_collection/terms/shard2
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 16 /solr/collections/collection1_collection/state.json
[junit4] 2> 9 /solr/clusterstate.json
[junit4] 2> 9 /solr/clusterprops.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 9 /solr/live_nodes
[junit4] 2> 9 /solr/collections
[junit4] 2>
[junit4] 2> 328579 INFO (SUITE-JdbcTest-seed#[AA2728CD153CBC80]-worker) [ ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
[junit4] 2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=false): {}, locale=es-DO, timezone=America/North_Dakota/Beulah
[junit4] 2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_172 (64-bit)/cpus=4,threads=1,free=176909592,total=276299776
[junit4] 2> NOTE: All tests run in this JVM: [FieldAnalysisResponseTest, HttpSolrClientConPoolTest, AnlysisResponseBaseTest, NaturalLogEvaluatorTest, CubedRootEvaluatorTest, RawValueEvaluatorTest, AndEvaluatorTest, SchemaTest, TestV1toV2ApiMapper, JdbcTest]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=JdbcTest -Dtests.seed=AA2728CD153CBC80 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.3/test-data/enwiki.random.lines.txt -Dtests.locale=es-DO -Dtests.timezone=America/North_Dakota/Beulah -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 0.00s J0 | JdbcTest (suite) <<<
[junit4] > Throwable #1: org.apache.solr.common.SolrException: Could not load collection from ZK: collection1_collection
[junit4] > at __randomizedtesting.SeedInfo.seed([AA2728CD153CBC80]:0)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1237)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:675)
[junit4] > at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:148)
[junit4] > at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:131)
[junit4] > at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:154)
[junit4] > at org.apache.solr.client.solrj.io.sql.JdbcTest.setupCluster(JdbcTest.java:78)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1_collection/state.json
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] > at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1249)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1235)
[junit4] > ... 29 more
[junit4] Completed [54/146 (1!)] on J0 in 74.47s, 0 tests, 1 error <<< FAILURES!
[...truncated 48911 lines...]