You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/01/16 04:27:07 UTC
[JENKINS] Lucene-Solr-Tests-7.x - Build # 316 - Failure
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/316/
9 tests failed.
FAILED: org.apache.solr.cloud.AddReplicaTest.test
Error Message:
core_node6:{"core":"addreplicatest_coll_shard1_replica_n5","base_url":"http://127.0.0.1:54964/solr","node_name":"127.0.0.1:54964_solr","state":"active","type":"NRT"}
Stack Trace:
java.lang.AssertionError: core_node6:{"core":"addreplicatest_coll_shard1_replica_n5","base_url":"http://127.0.0.1:54964/solr","node_name":"127.0.0.1:54964_solr","state":"active","type":"NRT"}
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:918A03DE6678FBE]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.AddReplicaTest.test(AddReplicaTest.java:84)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI
Error Message:
Timeout on query awhollynewcollection_0
Stack Trace:
java.lang.AssertionError: Timeout on query awhollynewcollection_0
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:C939EB534EA8CDD3]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:474)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.SystemLogListenerTest.test
Error Message:
wrong number of events added to .system expected:<9> but was:<10>
Stack Trace:
java.lang.AssertionError: wrong number of events added to .system expected:<9> but was:<10>
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:918A03DE6678FBE]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.apache.solr.cloud.autoscaling.SystemLogListenerTest.test(SystemLogListenerTest.java:166)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration
Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:43124_solr wasn't created
Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:43124_solr wasn't created
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:99F617EB46AE2FA9]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration(TriggerIntegrationTest.java:936)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode
Error Message:
did not finish processing changes
Stack Trace:
java.lang.AssertionError: did not finish processing changes
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:26A3824487D66D5E]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode(TestLargeCluster.java:297)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState
Error Message:
The trigger did not fire at all
Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:9711698725B03EB]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState(TestTriggerIntegration.java:365)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testEventQueue
Error Message:
action wasn't interrupted
Stack Trace:
java.lang.AssertionError: action wasn't interrupted
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:48F9DD4941FC24B3]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testEventQueue(TestTriggerIntegration.java:638)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testListeners
Error Message:
The trigger did not fire at all
Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:6C8C08F7F5F77F9F]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testListeners(TestTriggerIntegration.java:953)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testCooldown
Error Message:
CapturedEvent{timestamp=16382616708492790, stage=IGNORED, actionName='null', event={ "id":"3a33e64329c1b4T630hdbkyc23ofe1af80y30t32", "source":"node_added_cooldown_trigger", "eventTime":16382612711522740, "eventType":"NODEADDED", "properties":{ "eventTimes":[16382612711522740], "cooldown":true, "nodeNames":["127.0.0.1:10019_solr"]}}, context={}, config={ "trigger":"node_added_cooldown_trigger", "afterAction":[], "stage":[ "FAILED", "SUCCEEDED", "IGNORED"], "class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener", "beforeAction":[]}, message='In cooldown period.'} expected:<SUCCEEDED> but was:<IGNORED>
Stack Trace:
java.lang.AssertionError: CapturedEvent{timestamp=16382616708492790, stage=IGNORED, actionName='null', event={
"id":"3a33e64329c1b4T630hdbkyc23ofe1af80y30t32",
"source":"node_added_cooldown_trigger",
"eventTime":16382612711522740,
"eventType":"NODEADDED",
"properties":{
"eventTimes":[16382612711522740],
"cooldown":true,
"nodeNames":["127.0.0.1:10019_solr"]}}, context={}, config={
"trigger":"node_added_cooldown_trigger",
"afterAction":[],
"stage":[
"FAILED",
"SUCCEEDED",
"IGNORED"],
"class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",
"beforeAction":[]}, message='In cooldown period.'} expected:<SUCCEEDED> but was:<IGNORED>
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:B0F2F203363197B4]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testCooldown(TestTriggerIntegration.java:1107)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11847 lines...]
[junit4] Suite: org.apache.solr.cloud.AddReplicaTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AddReplicaTest_814C9FE7489BE246-001/init-core-data-001
[junit4] 2> 745985 INFO (SUITE-AddReplicaTest-seed#[814C9FE7489BE246]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 745996 INFO (SUITE-AddReplicaTest-seed#[814C9FE7489BE246]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 745996 INFO (SUITE-AddReplicaTest-seed#[814C9FE7489BE246]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 746012 INFO (SUITE-AddReplicaTest-seed#[814C9FE7489BE246]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AddReplicaTest_814C9FE7489BE246-001/tempDir-001
[junit4] 2> 746012 INFO (SUITE-AddReplicaTest-seed#[814C9FE7489BE246]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 746044 INFO (Thread-248) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 746087 INFO (Thread-248) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 746158 INFO (SUITE-AddReplicaTest-seed#[814C9FE7489BE246]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:44430
[junit4] 2> 746298 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
[junit4] 2> 746307 ERROR (Thread-248) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 747899 INFO (zkConnectionManagerCallback-133-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 747981 INFO (jetty-launcher-130-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 748016 INFO (jetty-launcher-130-thread-4) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 748021 INFO (jetty-launcher-130-thread-3) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 748021 INFO (jetty-launcher-130-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 748069 INFO (jetty-launcher-130-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4afdb32{/solr,null,AVAILABLE}
[junit4] 2> 748089 INFO (jetty-launcher-130-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@24c7f7c4{HTTP/1.1,[http/1.1]}{127.0.0.1:59586}
[junit4] 2> 748089 INFO (jetty-launcher-130-thread-1) [ ] o.e.j.s.Server Started @780042ms
[junit4] 2> 748089 INFO (jetty-launcher-130-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59586}
[junit4] 2> 748090 ERROR (jetty-launcher-130-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 748090 INFO (jetty-launcher-130-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 748090 INFO (jetty-launcher-130-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 748090 INFO (jetty-launcher-130-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 748090 INFO (jetty-launcher-130-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-16T01:04:09.794Z
[junit4] 2> 748625 INFO (jetty-launcher-130-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4becc214{/solr,null,AVAILABLE}
[junit4] 2> 748625 INFO (jetty-launcher-130-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@796968e2{HTTP/1.1,[http/1.1]}{127.0.0.1:40739}
[junit4] 2> 748625 INFO (jetty-launcher-130-thread-2) [ ] o.e.j.s.Server Started @780579ms
[junit4] 2> 748625 INFO (jetty-launcher-130-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40739}
[junit4] 2> 748626 ERROR (jetty-launcher-130-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 748626 INFO (jetty-launcher-130-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 748626 INFO (jetty-launcher-130-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 748626 INFO (jetty-launcher-130-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 748626 INFO (jetty-launcher-130-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-16T01:04:10.330Z
[junit4] 2> 748632 INFO (jetty-launcher-130-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7b5454ca{/solr,null,AVAILABLE}
[junit4] 2> 748632 INFO (jetty-launcher-130-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1c42ff6{HTTP/1.1,[http/1.1]}{127.0.0.1:54964}
[junit4] 2> 748633 INFO (jetty-launcher-130-thread-3) [ ] o.e.j.s.Server Started @780586ms
[junit4] 2> 748633 INFO (jetty-launcher-130-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=54964}
[junit4] 2> 748633 ERROR (jetty-launcher-130-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 748633 INFO (jetty-launcher-130-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 748633 INFO (jetty-launcher-130-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 748633 INFO (jetty-launcher-130-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 748633 INFO (jetty-launcher-130-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-16T01:04:10.337Z
[junit4] 2> 748716 INFO (jetty-launcher-130-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3805658f{/solr,null,AVAILABLE}
[junit4] 2> 748717 INFO (jetty-launcher-130-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@677d9fd1{HTTP/1.1,[http/1.1]}{127.0.0.1:49516}
[junit4] 2> 748717 INFO (jetty-launcher-130-thread-4) [ ] o.e.j.s.Server Started @780670ms
[junit4] 2> 748717 INFO (jetty-launcher-130-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=49516}
[junit4] 2> 748717 ERROR (jetty-launcher-130-thread-4) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 748717 INFO (jetty-launcher-130-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 748718 INFO (jetty-launcher-130-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 748718 INFO (jetty-launcher-130-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 748718 INFO (jetty-launcher-130-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-16T01:04:10.422Z
[junit4] 2> 748720 INFO (zkConnectionManagerCallback-135-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 748767 INFO (zkConnectionManagerCallback-137-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 748776 INFO (zkConnectionManagerCallback-139-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 748785 INFO (jetty-launcher-130-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 748808 INFO (jetty-launcher-130-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 748869 INFO (jetty-launcher-130-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 749000 INFO (zkConnectionManagerCallback-141-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 749043 INFO (jetty-launcher-130-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44430/solr
[junit4] 2> 749121 INFO (jetty-launcher-130-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 749953 INFO (jetty-launcher-130-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44430/solr
[junit4] 2> 749954 INFO (jetty-launcher-130-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44430/solr
[junit4] 2> 750017 INFO (zkConnectionManagerCallback-146-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 750237 INFO (jetty-launcher-130-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44430/solr
[junit4] 2> 750237 INFO (zkConnectionManagerCallback-151-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 750413 DEBUG (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$39/1242430876@1a0453cd
[junit4] 2> 750416 DEBUG (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$39/1242430876@6fe5548f
[junit4] 2> 750452 INFO (zkConnectionManagerCallback-153-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 750560 INFO (zkConnectionManagerCallback-157-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 750649 DEBUG (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$39/1242430876@68035e01
[junit4] 2> 750651 INFO (zkConnectionManagerCallback-161-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 750651 DEBUG (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$39/1242430876@1078f5dc
[junit4] 2> 750688 INFO (zkConnectionManagerCallback-159-thread-1-processing-n:127.0.0.1:54964_solr) [n:127.0.0.1:54964_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 750768 INFO (zkConnectionManagerCallback-165-thread-1-processing-n:127.0.0.1:40739_solr) [n:127.0.0.1:40739_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 750860 INFO (zkConnectionManagerCallback-163-thread-1-processing-n:127.0.0.1:49516_solr) [n:127.0.0.1:49516_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 752830 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 752832 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 752833 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 752835 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 752837 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 752839 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 752868 INFO (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 752873 DEBUG (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/99356812771393545-127.0.0.1:59586_solr-n_0000000000
[junit4] 2> 752880 INFO (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59586_solr
[junit4] 2> 752982 INFO (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.Overseer Overseer (id=99356812771393545-127.0.0.1:59586_solr-n_0000000000) starting
[junit4] 2> 753012 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 753016 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753038 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 753039 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 753042 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 753080 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 753120 INFO (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 753129 DEBUG (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/99356812771393547-127.0.0.1:49516_solr-n_0000000001
[junit4] 2> 753145 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/99356812771393545-127.0.0.1:59586_solr-n_0000000000
[junit4] 2> 753160 DEBUG (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/99356812771393545-127.0.0.1:59586_solr-n_0000000000 to know if I could be the leader
[junit4] 2> 753177 INFO (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49516_solr
[junit4] 2> 753200 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 753204 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 753301 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753301 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753347 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 753352 DEBUG (OverseerStateUpdate-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 753435 INFO (zkCallback-160-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 753435 INFO (zkCallback-162-thread-1-processing-n:127.0.0.1:49516_solr) [n:127.0.0.1:49516_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 753473 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 753476 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753476 INFO (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 753477 DEBUG (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:59586_solr as DOWN
[junit4] 2> 753480 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 753485 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 753492 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 753493 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 753508 INFO (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 753509 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 753512 DEBUG (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/99356812771393548-127.0.0.1:40739_solr-n_0000000002
[junit4] 2> 753512 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753513 INFO (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 753521 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 753522 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/99356812771393547-127.0.0.1:49516_solr-n_0000000001
[junit4] 2> 753536 DEBUG (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/99356812771393547-127.0.0.1:49516_solr-n_0000000001 to know if I could be the leader
[junit4] 2> 753536 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 753537 DEBUG (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:40739_solr as DOWN
[junit4] 2> 753537 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 753537 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
[junit4] 2> 753537 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 753537 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 753537 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 753538 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
[junit4] 2> 753538 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
[junit4] 2> 753540 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue-work
[junit4] 2> 753540 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 753557 INFO (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59586_solr
[junit4] 2> 753596 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 753609 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 753612 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 753621 INFO (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40739_solr
[junit4] 2> 753622 DEBUG (OverseerAutoScalingTriggerThread-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 753640 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 753641 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 753641 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 753641 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 753647 DEBUG (OverseerStateUpdate-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:59586_solr"} current state version: 0
[junit4] 2> 753647 DEBUG (OverseerStateUpdate-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:59586_solr
[junit4] 2> 753814 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753814 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753814 INFO (zkCallback-160-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
[junit4] 2> 753814 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/autoscaling.json
[junit4] 2> 753815 DEBUG (OverseerAutoScalingTriggerThread-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 753816 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753816 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753816 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 753817 DEBUG (OverseerAutoScalingTriggerThread-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:59586_solr, 127.0.0.1:40739_solr, 127.0.0.1:49516_solr]
[junit4] 2> 753817 DEBUG (OverseerAutoScalingTriggerThread-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
[junit4] 2> 753817 DEBUG (OverseerAutoScalingTriggerThread-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 753817 DEBUG (OverseerStateUpdate-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:40739_solr"} current state version: 0
[junit4] 2> 753818 DEBUG (OverseerStateUpdate-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:40739_solr
[junit4] 2> 753846 INFO (zkCallback-162-thread-1-processing-n:127.0.0.1:49516_solr) [n:127.0.0.1:49516_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
[junit4] 2> 753932 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 753952 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 754013 INFO (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 754014 DEBUG (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/99356812771393546-127.0.0.1:54964_solr-n_0000000003
[junit4] 2> 754032 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/99356812771393548-127.0.0.1:40739_solr-n_0000000002
[junit4] 2> 754032 DEBUG (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/99356812771393548-127.0.0.1:40739_solr-n_0000000002 to know if I could be the leader
[junit4] 2> 754032 DEBUG (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:54964_solr as DOWN
[junit4] 2> 754034 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 754035 INFO (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54964_solr
[junit4] 2> 754035 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 754036 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 754036 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 754036 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 754052 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 754052 INFO (zkCallback-162-thread-1-processing-n:127.0.0.1:49516_solr) [n:127.0.0.1:49516_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 754077 DEBUG (OverseerAutoScalingTriggerThread-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 754078 DEBUG (OverseerAutoScalingTriggerThread-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 754096 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 754116 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 754136 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 754137 INFO (zkCallback-160-thread-2-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 754212 INFO (zkCallback-158-thread-1-processing-n:127.0.0.1:54964_solr) [n:127.0.0.1:54964_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (4)
[junit4] 2> 754212 INFO (zkCallback-164-thread-1-processing-n:127.0.0.1:40739_solr) [n:127.0.0.1:40739_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
[junit4] 2> 754097 DEBUG (ScheduledTrigger-172-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 754532 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 754533 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 754552 INFO (zkCallback-164-thread-2-processing-n:127.0.0.1:40739_solr) [n:127.0.0.1:40739_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 754564 DEBUG (OverseerStateUpdate-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:54964_solr"} current state version: 0
[junit4] 2> 754564 DEBUG (OverseerStateUpdate-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:54964_solr
[junit4] 2> 754791 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 754797 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 755556 DEBUG (ScheduledTrigger-172-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 755896 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
[junit4] 2> 755897 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
[junit4] 2> 755897 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 755897 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 755897 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 755897 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
[junit4] 2> 755897 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
[junit4] 2> 758350 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
[junit4] 2> 758352 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
[junit4] 2> 758352 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 758352 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 758352 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 758352 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
[junit4] 2> 758352 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
[junit4] 2> 758572 DEBUG (ScheduledTrigger-172-thread-2) [n:127.0.0.1:59586_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 758981 INFO (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54964.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 758984 INFO (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49516.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 759345 INFO (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54964.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 759346 INFO (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54964.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 759377 INFO (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40739.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 759421 INFO (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AddReplicaTest_814C9FE7489BE246-001/tempDir-001/node3/.
[junit4] 2> 759422 INFO (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49516.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 759422 INFO (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49516.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 759596 DEBUG (ScheduledTrigger-172-thread-2) [n:127.0.0.1:59586_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 759625 INFO (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59586.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 759628 INFO (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AddReplicaTest_814C9FE7489BE246-001/tempDir-001/node4/.
[junit4] 2> 760060 INFO (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40739.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 760061 INFO (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40739.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 760078 INFO (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59586.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 760078 INFO (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59586.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 760170 INFO (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AddReplicaTest_814C9FE7489BE246-001/tempDir-001/node2/.
[junit4] 2> 760211 INFO (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AddReplicaTest_814C9FE7489BE246-001/tempDir-001/node1/.
[junit4] 2> 760352 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
[junit4] 2> 760353 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
[junit4] 2> 760353 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 760353 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 760353 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 760353 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
[junit4] 2> 760353 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
[junit4] 2> 760597 DEBUG (ScheduledTrigger-172-thread-2) [n:127.0.0.1:59586_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 761598 DEBUG (ScheduledTrigger-172-thread-2) [n:127.0.0.1:59586_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 762353 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
[junit4] 2> 762354 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
[junit4] 2> 762354 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 762354 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 762354 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 762354 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
[junit4] 2> 762354 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
[junit4] 2> 762482 INFO (zkConnectionManagerCallback-175-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 762572 INFO (zkConnectionManagerCallback-179-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 762573 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 762582 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 762582 INFO (SUITE-AddReplicaTest-seed#[814C9FE7489BE246]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 762582 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 762582 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 762592 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 762593 INFO (SUITE-AddReplicaTest-seed#[814C9FE7489BE246]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44430/solr ready
[junit4] 2> 762604 DEBUG (ScheduledTrigger-172-thread-2) [n:127.0.0.1:59586_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 762833 INFO (TEST-AddReplicaTest.test-seed#[814C9FE7489BE246]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 762871 INFO (zkConnectionManagerCallback-181-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 762965 INFO (qtp1811337134-765) [n:127.0.0.1:49516_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&maxShardsPerNode=2&name=addreplicatest_coll&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 762968 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 762976 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/overseer/collection-queue-work/qnr-0000000000
[junit4] 2> 762976 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 762976 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000000
[junit4] 2> 762977 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 762977 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 762984 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000000
[junit4] 2> 762984 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskQueue Returning topN elements: [/overseer/collection-queue-work/qn-0000000000, ]
[junit4] 2> 762984 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Got 1 tasks from work-queue : [[org.apache.solr.cloud.OverseerTaskQueue$QueueEvent@2e684ecf]]
[junit4] 2> 762984 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as running
[junit4] 2> 762985 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
[junit4] 2> "name":"addreplicatest_coll",
[junit4] 2> "fromApi":"true",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"2",
[junit4] 2> "maxShardsPerNode":"2",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "operation":"create"}
[junit4] 2> 762985 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1
[junit4] 2> 762985 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 762985 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 762985 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 762985 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 762985 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 99 elements. ExcludeSet: [/overseer/collection-queue-work/qn-0000000000],[]
[junit4] 2> 763014 DEBUG (OverseerThreadFactory-174-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Runner processing /overseer/collection-queue-work/qn-0000000000
[junit4] 2> 763014 DEBUG (OverseerThreadFactory-174-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
[junit4] 2> "name":"addreplicatest_coll",
[junit4] 2> "fromApi":"true",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"2",
[junit4] 2> "maxShardsPerNode":"2",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "operation":"create"}
[junit4] 2> 763014 INFO (OverseerThreadFactory-174-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.CreateCollectionCmd Create collection addreplicatest_coll
[junit4] 2> 763024 DEBUG (OverseerThreadFactory-174-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.Assign Identify nodes using default
[junit4] 2> 763024 DEBUG (OverseerThreadFactory-174-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/addreplicatest_coll
[junit4] 2> 763033 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 763033 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 763033 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 763033 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 763033 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 763040 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 763040 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 763044 DEBUG (OverseerThreadFactory-174-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.CreateCollectionCmd Check for collection zkNode:addreplicatest_coll
[junit4] 2> 763044 DEBUG (OverseerThreadFactory-174-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.CreateCollectionCmd Collection zkNode exists
[junit4] 2> 763052 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 763054 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 763054 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 763054 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 763055 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 763056 DEBUG (OverseerStateUpdate-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "name":"addreplicatest_coll",
[junit4] 2> "fromApi":"true",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "collection.c
[...truncated too long message...]
ve/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptor-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-admin-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-authn-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-authz-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/bcprov-jdk15on-1.54.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/byte-buddy-1.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/commons-math3-3.6.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/mockito-core-2.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/objenesis-2.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/analysis/icu/lucene-analyzers-icu-7.3.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/backward-codecs/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/contrib/analysis-extras/lib/icu4j-60.2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.4.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_144/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J1-20180116_005109_5405703862273861884410.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J1-20180116_005109_540569789209671531803.suites -stdin
[junit4] ERROR: JVM J1 ended with an exception: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J1-20180116_005109_5401070842046531995164.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J1-20180116_005109_5404755002079522906066.syserr
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1519)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
[junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] at java.lang.Thread.run(Thread.java:748)
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:836: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:780: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build.xml:262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/common-build.xml:556: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1513: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1038: At least one slave process threw an exception, first: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J1-20180116_005109_5401070842046531995164.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J1-20180116_005109_5404755002079522906066.syserr
Total time: 395 minutes 49 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-Tests-7.x #315 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 320 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/320/
5 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestBlobHandler
Error Message:
38 threads leaked from SUITE scope at org.apache.solr.handler.TestBlobHandler: 1) Thread[id=6903, name=qtp258116223-6903, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Object.wait(Native Method) at org.apache.solr.cloud.OverseerTaskQueue$LatchWatcher.await(OverseerTaskQueue.java:149) at org.apache.solr.cloud.OverseerTaskQueue.offer(OverseerTaskQueue.java:199) at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:306) at org.apache.solr.handler.admin.CollectionsHandler.invokeAction(CollectionsHandler.java:246) at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:224) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:527) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:530) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.io.ManagedSelector$$Lambda$24/1133556656.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=6914, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=6909, name=qtp258116223-6909, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=6943, name=zkConnectionManagerCallback-884-thread-1-processing-n:127.0.0.1:36328_wv%2Fx-EventThread, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 5) Thread[id=6975, name=zkCallback-883-thread-6, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=6944, name=OverseerStateUpdate-99373310955421706-127.0.0.1:36328_wv%2Fx-n_0000000002, state=TIMED_WAITING, group=Overseer state updater.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:339) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:415) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=6906, name=qtp258116223-6906, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=6935, name=zkCallback-883-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=6941, name=zkCallback-883-thread-5, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=6968, name=updateExecutor-879-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=6925, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=6947, name=ScheduledTrigger-2312-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=6976, name=zkCallback-883-thread-7, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=6905, name=qtp258116223-6905, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=6945, name=OverseerCollectionConfigSetProcessor-99373310955421706-127.0.0.1:36328_wv%2Fx-n_0000000002, state=TIMED_WAITING, group=Overseer collection creation process.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:339) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:392) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=6942, name=zkConnectionManagerCallback-884-thread-1-processing-n:127.0.0.1:36328_wv%2Fx-SendThread(127.0.0.1:42967), state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 17) Thread[id=6908, name=qtp258116223-6908, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=6902, name=qtp258116223-6902, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=6922, name=zkCallback-883-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=6959, name=ScheduledTrigger-2312-thread-3, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=6930, name=watches-885-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=6969, name=recoveryExecutor-880-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=6950, name=coreZkRegister-2300-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=6932, name=searcherExecutor-2306-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=6907, name=qtp258116223-6907, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=6967, name=searcherExecutor-2315-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=6904, name=qtp258116223-6904-acceptor-0@360e97d4-ServerConnector@3fe8a307{HTTP/1.1,[http/1.1]}{127.0.0.1:36328}, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 28) Thread[id=6915, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=6921, name=zkConnectionManagerCallback-884-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=6926, name=Scheduler-1674259162, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 31) Thread[id=6965, name=ScheduledTrigger-2312-thread-4, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=6910, name=Scheduler-411863045, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=6923, name=Thread-2986, state=WAITING, group=TGRP-TestBlobHandler] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1722) 34) Thread[id=6949, name=OverseerThreadFactory-2314-thread-1, state=WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=6936, name=zkCallback-883-thread-3, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=6948, name=ScheduledTrigger-2312-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:315) at org.apache.solr.client.solrj.impl.ZkDistribStateManager.hasData(ZkDistribStateManager.java:57) at org.apache.solr.cloud.autoscaling.TriggerBase.saveState(TriggerBase.java:203) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.run(ScheduledTriggers.java:544) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 37) Thread[id=6946, name=OverseerAutoScalingTriggerThread-99373310955421706-127.0.0.1:36328_wv%2Fx-n_0000000002, state=WAITING, group=Overseer autoscaling triggers] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166) at java.lang.Thread.run(Thread.java:748) 38) Thread[id=6937, name=zkCallback-883-thread-4, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 38 threads leaked from SUITE scope at org.apache.solr.handler.TestBlobHandler:
1) Thread[id=6903, name=qtp258116223-6903, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at java.lang.Object.wait(Native Method)
at org.apache.solr.cloud.OverseerTaskQueue$LatchWatcher.await(OverseerTaskQueue.java:149)
at org.apache.solr.cloud.OverseerTaskQueue.offer(OverseerTaskQueue.java:199)
at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:306)
at org.apache.solr.handler.admin.CollectionsHandler.invokeAction(CollectionsHandler.java:246)
at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:224)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:527)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at org.eclipse.jetty.server.Server.handle(Server.java:530)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$24/1133556656.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=6914, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=6909, name=qtp258116223-6909, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=6943, name=zkConnectionManagerCallback-884-thread-1-processing-n:127.0.0.1:36328_wv%2Fx-EventThread, state=WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
5) Thread[id=6975, name=zkCallback-883-thread-6, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=6944, name=OverseerStateUpdate-99373310955421706-127.0.0.1:36328_wv%2Fx-n_0000000002, state=TIMED_WAITING, group=Overseer state updater.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:339)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:415)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=6906, name=qtp258116223-6906, state=RUNNABLE, group=TGRP-TestBlobHandler]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=6935, name=zkCallback-883-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=6941, name=zkCallback-883-thread-5, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=6968, name=updateExecutor-879-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=6925, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=6947, name=ScheduledTrigger-2312-thread-1, state=WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=6976, name=zkCallback-883-thread-7, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=6905, name=qtp258116223-6905, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=6945, name=OverseerCollectionConfigSetProcessor-99373310955421706-127.0.0.1:36328_wv%2Fx-n_0000000002, state=TIMED_WAITING, group=Overseer collection creation process.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:339)
at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:392)
at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)
at java.lang.Thread.run(Thread.java:748)
16) Thread[id=6942, name=zkConnectionManagerCallback-884-thread-1-processing-n:127.0.0.1:36328_wv%2Fx-SendThread(127.0.0.1:42967), state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
17) Thread[id=6908, name=qtp258116223-6908, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
18) Thread[id=6902, name=qtp258116223-6902, state=RUNNABLE, group=TGRP-TestBlobHandler]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
19) Thread[id=6922, name=zkCallback-883-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
20) Thread[id=6959, name=ScheduledTrigger-2312-thread-3, state=WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
21) Thread[id=6930, name=watches-885-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
22) Thread[id=6969, name=recoveryExecutor-880-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
23) Thread[id=6950, name=coreZkRegister-2300-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
24) Thread[id=6932, name=searcherExecutor-2306-thread-1, state=WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
25) Thread[id=6907, name=qtp258116223-6907, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
26) Thread[id=6967, name=searcherExecutor-2315-thread-1, state=WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
27) Thread[id=6904, name=qtp258116223-6904-acceptor-0@360e97d4-ServerConnector@3fe8a307{HTTP/1.1,[http/1.1]}{127.0.0.1:36328}, state=RUNNABLE, group=TGRP-TestBlobHandler]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
28) Thread[id=6915, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
29) Thread[id=6921, name=zkConnectionManagerCallback-884-thread-1, state=WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
30) Thread[id=6926, name=Scheduler-1674259162, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
31) Thread[id=6965, name=ScheduledTrigger-2312-thread-4, state=WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
32) Thread[id=6910, name=Scheduler-411863045, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
33) Thread[id=6923, name=Thread-2986, state=WAITING, group=TGRP-TestBlobHandler]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1722)
34) Thread[id=6949, name=OverseerThreadFactory-2314-thread-1, state=WAITING, group=Overseer collection creation process.]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
35) Thread[id=6936, name=zkCallback-883-thread-3, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
36) Thread[id=6948, name=ScheduledTrigger-2312-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:315)
at org.apache.solr.client.solrj.impl.ZkDistribStateManager.hasData(ZkDistribStateManager.java:57)
at org.apache.solr.cloud.autoscaling.TriggerBase.saveState(TriggerBase.java:203)
at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.run(ScheduledTriggers.java:544)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
37) Thread[id=6946, name=OverseerAutoScalingTriggerThread-99373310955421706-127.0.0.1:36328_wv%2Fx-n_0000000002, state=WAITING, group=Overseer autoscaling triggers]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166)
at java.lang.Thread.run(Thread.java:748)
38) Thread[id=6937, name=zkCallback-883-thread-4, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([D9ED50908F29A796]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestBlobHandler
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=6909, name=qtp258116223-6909, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=6975, name=zkCallback-883-thread-6, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=6906, name=qtp258116223-6906, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=6935, name=zkCallback-883-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=6941, name=zkCallback-883-thread-5, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=6968, name=updateExecutor-879-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=6947, name=ScheduledTrigger-2312-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=6976, name=zkCallback-883-thread-7, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=6905, name=qtp258116223-6905, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=6942, name=zkConnectionManagerCallback-884-thread-1-processing-n:127.0.0.1:36328_wv%2Fx-SendThread(127.0.0.1:42967), state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 11) Thread[id=6902, name=qtp258116223-6902, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=6922, name=zkCallback-883-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=6959, name=ScheduledTrigger-2312-thread-3, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=6930, name=watches-885-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=6969, name=recoveryExecutor-880-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=6950, name=coreZkRegister-2300-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=6932, name=searcherExecutor-2306-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=6907, name=qtp258116223-6907, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=6967, name=searcherExecutor-2315-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=6921, name=zkConnectionManagerCallback-884-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=6926, name=Scheduler-1674259162, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=6965, name=ScheduledTrigger-2312-thread-4, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=6910, name=Scheduler-411863045, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=6923, name=Thread-2986, state=WAITING, group=TGRP-TestBlobHandler] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1722) 25) Thread[id=6936, name=zkCallback-883-thread-3, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=6948, name=ScheduledTrigger-2312-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=6937, name=zkCallback-883-thread-4, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=6909, name=qtp258116223-6909, state=RUNNABLE, group=TGRP-TestBlobHandler]
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=6975, name=zkCallback-883-thread-6, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concu
[...truncated too long message...]
ask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 18) Thread[id=6907, name=qtp258116223-6907, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
[junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
[junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 19) Thread[id=6967, name=searcherExecutor-2315-thread-1, state=WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 20) Thread[id=6921, name=zkConnectionManagerCallback-884-thread-1, state=WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 21) Thread[id=6926, name=Scheduler-1674259162, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 22) Thread[id=6965, name=ScheduledTrigger-2312-thread-4, state=WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 23) Thread[id=6910, name=Scheduler-411863045, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 24) Thread[id=6923, name=Thread-2986, state=WAITING, group=TGRP-TestBlobHandler]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1722)
[junit4] > 25) Thread[id=6936, name=zkCallback-883-thread-3, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 26) Thread[id=6948, name=ScheduledTrigger-2312-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 27) Thread[id=6937, name=zkCallback-883-thread-4, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([D9ED50908F29A796]:0)
[junit4] Completed [263/767 (3!)] on J2 in 109.37s, 1 test, 3 errors <<< FAILURES!
[...truncated 49708 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 319 - Still unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/319/
6 tests failed.
FAILED: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test
Error Message:
Timed out waiting for replica core_node54 (1516235137951) to replicate from leader core_node46 (0)
Stack Trace:
java.lang.AssertionError: Timed out waiting for replica core_node54 (1516235137951) to replicate from leader core_node46 (0)
at __randomizedtesting.SeedInfo.seed([4384683B4A8E81CC:CBD057E1E472EC34]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForReplicationFromReplicas(AbstractFullDistribZkTestBase.java:2143)
at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:268)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
Error Message:
9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest: 1) Thread[id=1789, name=zkCallback-292-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=1746, name=zkCallback-292-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=1414, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=1415, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]-SendThread(127.0.0.1:40492), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 5) Thread[id=1417, name=zkConnectionManagerCallback-293-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=1747, name=zkCallback-292-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=1920, name=zkCallback-292-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=1416, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 9) Thread[id=1419, name=zkCallback-292-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest:
1) Thread[id=1789, name=zkCallback-292-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=1746, name=zkCallback-292-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=1414, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=1415, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]-SendThread(127.0.0.1:40492), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
5) Thread[id=1417, name=zkConnectionManagerCallback-293-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=1747, name=zkCallback-292-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=1920, name=zkCallback-292-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=1416, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
9) Thread[id=1419, name=zkCallback-292-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([4384683B4A8E81CC]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=1789, name=zkCallback-292-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=1746, name=zkCallback-292-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=1415, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]-SendThread(127.0.0.1:40492), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 4) Thread[id=1417, name=zkConnectionManagerCallback-293-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=1747, name=zkCallback-292-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=1920, name=zkCallback-292-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=1419, name=zkCallback-292-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=1789, name=zkCallback-292-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=1746, name=zkCallback-292-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=1415, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]-SendThread(127.0.0.1:40492), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
4) Thread[id=1417, name=zkConnectionManagerCallback-293-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=1747, name=zkCallback-292-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=1920, name=zkCallback-292-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=1419, name=zkCallback-292-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([4384683B4A8E81CC]:0)
FAILED: org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory
Error Message:
expected:<8> but was:<9>
Stack Trace:
java.lang.AssertionError: expected:<8> but was:<9>
at __randomizedtesting.SeedInfo.seed([4384683B4A8E81CC:2E78CCC6F0C67ECB]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:259)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessorTest.test
Error Message:
Expected no errors: [{type=ADD,id=1,message=Unexpected type of routeKey: null}]
Stack Trace:
java.lang.AssertionError: Expected no errors: [{type=ADD,id=1,message=Unexpected type of routeKey: null}]
at __randomizedtesting.SeedInfo.seed([4384683B4A8E81CC:CBD057E1E472EC34]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessorTest.assertUpdateResponse(TimeRoutedAliasUpdateProcessorTest.java:270)
at org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessorTest.test(TimeRoutedAliasUpdateProcessorTest.java:132)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.common.cloud.TestCollectionStateWatchers.testPredicateFailureTimesOut
Error Message:
Watchers for collection should be removed after timeout
Stack Trace:
java.lang.AssertionError: Watchers for collection should be removed after timeout
at __randomizedtesting.SeedInfo.seed([9EDBA3692A4378C6:63E2C555735BADBC]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.common.cloud.TestCollectionStateWatchers.waitFor(TestCollectionStateWatchers.java:107)
at org.apache.solr.common.cloud.TestCollectionStateWatchers.testPredicateFailureTimesOut(TestCollectionStateWatchers.java:216)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11872 lines...]
[junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/init-core-data-001
[junit4] 2> 657073 WARN (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[4384683B4A8E81CC]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
[junit4] 2> 657074 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[4384683B4A8E81CC]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 657110 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[4384683B4A8E81CC]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 657110 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[4384683B4A8E81CC]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 657110 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[4384683B4A8E81CC]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /oi/pl
[junit4] 2> 657111 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.ChaosMonkeyNothingIsSafeWithPullReplicasTest Starting ChaosMonkey test with 2 shards and 8 nodes
[junit4] 2> 657231 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 657232 INFO (Thread-235) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 657232 INFO (Thread-235) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 657234 ERROR (Thread-235) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 657332 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.ZkTestServer start zk server on port:40492
[junit4] 2> 657333 INFO (zkConnectionManagerCallback-273-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 657459 INFO (zkConnectionManagerCallback-275-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 657470 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 657481 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 657490 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 657491 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 657510 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 657511 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 657512 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 657513 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 657552 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 657559 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 657566 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 657695 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 658691 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T22:27:37+01:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 658758 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 658758 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 658759 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 658759 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@66c7a66a{/oi/pl,null,AVAILABLE}
[junit4] 2> 658769 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@51d464c2{HTTP/1.1,[http/1.1]}{127.0.0.1:45771}
[junit4] 2> 658769 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.Server Started @686706ms
[junit4] 2> 658769 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/tempDir-001/control/data, replicaType=NRT, hostContext=/oi/pl, hostPort=45771, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/control-001/cores}
[junit4] 2> 658769 ERROR (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 658769 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 658769 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 658769 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 658769 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-18T00:22:31.182Z
[junit4] 2> 658887 INFO (zkConnectionManagerCallback-277-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 658889 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 658889 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/control-001/solr.xml
[junit4] 2> 658911 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 658911 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 658912 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 658942 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40492/solr
[junit4] 2> 658967 INFO (zkConnectionManagerCallback-281-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 659058 INFO (zkConnectionManagerCallback-283-thread-1-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 659769 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 659770 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45771_oi%2Fpl
[junit4] 2> 659770 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.Overseer Overseer (id=99367973660000260-127.0.0.1:45771_oi%2Fpl-n_0000000000) starting
[junit4] 2> 659898 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45771_oi%2Fpl
[junit4] 2> 659941 INFO (zkCallback-282-thread-1-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 662044 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b
[junit4] 2> 662354 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b
[junit4] 2> 662354 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b
[junit4] 2> 662357 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/control-001/cores
[junit4] 2> 662786 INFO (zkConnectionManagerCallback-289-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 662806 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 662807 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40492/solr ready
[junit4] 2> 662809 INFO (qtp656415825-1370) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45771_oi%252Fpl&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 662931 INFO (OverseerThreadFactory-519-thread-1-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 663403 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 663404 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 663518 INFO (zkCallback-282-thread-1-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 664866 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 665139 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 666504 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 666910 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 666910 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b
[junit4] 2> 666911 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 666911 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 666914 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 667101 WARN (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 668049 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 668049 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 668051 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 668052 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 668095 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=38.2451171875, floorSegmentMB=1.5830078125, forceMergeDeletesPctAllowed=21.032870952662417, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 668214 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@64667f4[control_collection_shard1_replica_n1] main]
[junit4] 2> 668224 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 668226 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 668227 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 668247 INFO (searcherExecutor-522-thread-1-processing-n:127.0.0.1:45771_oi%2Fpl x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@64667f4[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 668248 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589887590110068736
[junit4] 2> 668328 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 668328 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 668328 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45771/oi/pl/control_collection_shard1_replica_n1/
[junit4] 2> 668329 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 668329 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:45771/oi/pl/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 668329 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 668356 INFO (zkCallback-282-thread-1-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 668387 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45771/oi/pl/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 668387 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 668426 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=5023
[junit4] 2> 668472 INFO (qtp656415825-1370) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 668490 INFO (zkCallback-282-thread-1-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 669472 INFO (qtp656415825-1370) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45771_oi%252Fpl&wt=javabin&version=2} status=0 QTime=6664
[junit4] 2> 669954 INFO (zkConnectionManagerCallback-293-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 669955 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 669957 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40492/solr ready
[junit4] 2> 669957 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 669963 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 670304 INFO (OverseerThreadFactory-519-thread-2-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 670346 INFO (OverseerCollectionConfigSetProcessor-99367973660000260-127.0.0.1:45771_oi%2Fpl-n_0000000000) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 670386 WARN (OverseerThreadFactory-519-thread-2-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 670620 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 670620 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=657
[junit4] 2> 671948 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/shard-1-001 of type NRT
[junit4] 2> 671963 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T22:27:37+01:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 672094 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 672094 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 672095 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 672095 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@29ee3f0f{/oi/pl,null,AVAILABLE}
[junit4] 2> 672095 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4d8bf561{HTTP/1.1,[http/1.1]}{127.0.0.1:36274}
[junit4] 2> 672095 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.Server Started @700033ms
[junit4] 2> 672095 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/oi/pl, hostPort=36274, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/shard-1-001/cores}
[junit4] 2> 672096 ERROR (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 672096 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 672096 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 672096 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 672096 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-18T00:22:44.509Z
[junit4] 2> 672330 INFO (zkConnectionManagerCallback-295-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 672338 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 672338 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/shard-1-001/solr.xml
[junit4] 2> 672342 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 672342 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 672398 INFO (OverseerCollectionConfigSetProcessor-99367973660000260-127.0.0.1:45771_oi%2Fpl-n_0000000000) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 672423 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 672543 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40492/solr
[junit4] 2> 672666 INFO (zkConnectionManagerCallback-299-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 672676 INFO (zkConnectionManagerCallback-301-thread-1-processing-n:127.0.0.1:36274_oi%2Fpl) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 672918 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 672920 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 672939 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36274_oi%2Fpl
[junit4] 2> 672939 INFO (zkCallback-282-thread-1-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 672940 INFO (zkCallback-292-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 673178 INFO (zkCallback-300-thread-1-processing-n:127.0.0.1:36274_oi%2Fpl) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 674377 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b
[junit4] 2> 674605 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b
[junit4] 2> 674605 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b
[junit4] 2> 674628 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/shard-1-001/cores
[junit4] 2> 675095 INFO (qtp308972610-1421) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:36274_oi%252Fpl&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 676167 INFO (OverseerThreadFactory-519-thread-3-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:36274_oi%2Fpl for creating new replica
[junit4] 2> 676624 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 676625 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 677981 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 678179 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema [collection1_shard2_replica_n41] Schema name=test
[junit4] 2> 680660 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 681019 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n41' using configuration from collection collection1, trusted=true
[junit4] 2> 681020 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n41' (registry 'solr.core.collection1.shard2.replica_n41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b
[junit4] 2> 681020 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 681020 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [[collection1_shard2_replica_n41] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/shard-1-001/cores/collection1_shard2_replica_n41], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/shard-1-001/cores/collection1_shard2_replica_n41/data/]
[junit4] 2> 681099 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 681802 WARN (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 682154 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 682154 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 682155 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 682155 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 682156 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=38.2451171875, floorSegmentMB=1.5830078125, forceMergeDeletesPctAllowed=21.032870952662417, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 682218 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.SolrIndexSearcher Opening [Searcher@16ac7a2a[collection1_shard2_replica_n41] main]
[junit4] 2> 682838 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 682845 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 682846 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 682846 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589887605417181184
[junit4] 2> 682855 INFO (searcherExecutor-533-thread-1-processing-n:127.0.0.1:36274_oi%2Fpl x:collection1_shard2_replica_n41 s:shard2 c:collection1) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [collection1_shard2_replica_n41] Registered new searcher Searcher@16ac7a2a[collection1_shard2_replica_n41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 682879 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 682879 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 682879 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:36274/oi/pl/collection1_shard2_replica_n41/
[junit4] 2> 682879 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 682879 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy http://127.0.0.1:36274/oi/pl/collection1_shard2_replica_n41/ has no replicas
[junit4] 2> 682879 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 682896 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:36274/oi/pl/collection1_shard2_replica_n41/ shard2
[junit4] 2> 683222 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 683223 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=6599
[junit4] 2> 683247 INFO (qtp308972610-1421) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:36274_oi%252Fpl&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=8151
[junit4] 2> 683755 INFO (OverseerCollectionConfigSetProcessor-99367973660000260-127.0.0.1:45771_oi%2Fpl-n_0000000000) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 686605 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/shard-2-001 of type NRT
[junit4] 2> 686691 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T22:27:37+01:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 686710 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 686710 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 686711 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 686711 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@41a87c3d{/oi/pl,null,AVAILABLE}
[junit4] 2> 686711 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5b9afbdd{HTTP/1.1,[http/1.1]}{127.0.0.1:40228}
[junit4] 2> 686711 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.Server Started @714649ms
[junit4] 2> 686711 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/tempDir-001/jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/oi/pl, hostPort=40228, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/L
[...truncated too long message...]
20 INFO (coreCloseExecutor-161-thread-2) [n:127.0.0.1:58516_solr c:delayed s:shard1 r:core_node2 x:delayed_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.delayed.shard1.leader, tag=813847959
[junit4] 2> 407320 INFO (coreCloseExecutor-161-thread-3) [n:127.0.0.1:58516_solr c:testcollection s:shard1 r:core_node3 x:testcollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testcollection.shard1.replica_n1, tag=1164830252
[junit4] 2> 407320 INFO (coreCloseExecutor-161-thread-3) [n:127.0.0.1:58516_solr c:testcollection s:shard1 r:core_node3 x:testcollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@61b12365: rootName = solr_58516, domain = solr.core.testcollection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.testcollection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@39cab994
[junit4] 2> 407340 INFO (coreCloseExecutor-162-thread-1) [n:127.0.0.1:59305_solr c:falsepredicate s:shard1 r:core_node2 x:falsepredicate_shard1_replica_n1] o.a.s.c.SolrCore [falsepredicate_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@2912a4ca
[junit4] 2> 407340 INFO (coreCloseExecutor-162-thread-1) [n:127.0.0.1:59305_solr c:falsepredicate s:shard1 r:core_node2 x:falsepredicate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.falsepredicate.shard1.replica_n1, tag=689087690
[junit4] 2> 407340 INFO (coreCloseExecutor-162-thread-1) [n:127.0.0.1:59305_solr c:falsepredicate s:shard1 r:core_node2 x:falsepredicate_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@19d5a472: rootName = solr_59305, domain = solr.core.falsepredicate.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.falsepredicate.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@21105b4d
[junit4] 2> 407432 INFO (coreCloseExecutor-161-thread-4) [n:127.0.0.1:58516_solr c:waitforstate s:shard1 r:core_node2 x:waitforstate_shard1_replica_n1] o.a.s.c.SolrCore [waitforstate_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@3efc472e
[junit4] 2> 407510 INFO (coreCloseExecutor-161-thread-4) [n:127.0.0.1:58516_solr c:waitforstate s:shard1 r:core_node2 x:waitforstate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.waitforstate.shard1.replica_n1, tag=1056720686
[junit4] 2> 407510 INFO (coreCloseExecutor-162-thread-1) [n:127.0.0.1:59305_solr c:falsepredicate s:shard1 r:core_node2 x:falsepredicate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.falsepredicate.shard1.leader, tag=689087690
[junit4] 2> 407510 INFO (coreCloseExecutor-162-thread-2) [n:127.0.0.1:59305_solr c:testcollection s:shard3 r:core_node7 x:testcollection_shard3_replica_n4] o.a.s.c.SolrCore [testcollection_shard3_replica_n4] CLOSING SolrCore org.apache.solr.core.SolrCore@12043cdc
[junit4] 2> 407510 INFO (coreCloseExecutor-162-thread-2) [n:127.0.0.1:59305_solr c:testcollection s:shard3 r:core_node7 x:testcollection_shard3_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testcollection.shard3.replica_n4, tag=302267612
[junit4] 2> 407510 INFO (coreCloseExecutor-162-thread-2) [n:127.0.0.1:59305_solr c:testcollection s:shard3 r:core_node7 x:testcollection_shard3_replica_n4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5fd5a81c: rootName = solr_59305, domain = solr.core.testcollection.shard3.replica_n4, service url = null, agent id = null] for registry solr.core.testcollection.shard3.replica_n4 / com.codahale.metrics.MetricRegistry@24e2995d
[junit4] 2> 407728 INFO (coreCloseExecutor-161-thread-4) [n:127.0.0.1:58516_solr c:waitforstate s:shard1 r:core_node2 x:waitforstate_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7b53ffd8: rootName = solr_58516, domain = solr.core.waitforstate.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.waitforstate.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@2722b1e
[junit4] 2> 407865 INFO (coreCloseExecutor-162-thread-2) [n:127.0.0.1:59305_solr c:testcollection s:shard3 r:core_node7 x:testcollection_shard3_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testcollection.shard3.leader, tag=302267612
[junit4] 2> 407865 INFO (coreCloseExecutor-161-thread-3) [n:127.0.0.1:58516_solr c:testcollection s:shard1 r:core_node3 x:testcollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testcollection.shard1.leader, tag=1164830252
[junit4] 2> 407866 INFO (coreCloseExecutor-161-thread-4) [n:127.0.0.1:58516_solr c:waitforstate s:shard1 r:core_node2 x:waitforstate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.waitforstate.shard1.leader, tag=1056720686
[junit4] 2> 407964 INFO (jetty-closer-13-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7a14c812{/solr,null,UNAVAILABLE}
[junit4] 2> 407980 INFO (jetty-closer-13-thread-3) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 409428 INFO (jetty-closer-13-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@400b5127{/solr,null,UNAVAILABLE}
[junit4] 2> 409429 INFO (jetty-closer-13-thread-4) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 409476 INFO (jetty-closer-13-thread-1) [ ] o.a.s.c.Overseer Overseer (id=99368291327213580-127.0.0.1:59305_solr-n_0000000003) closing
[junit4] 2> 409476 INFO (OverseerStateUpdate-99368291327213580-127.0.0.1:59305_solr-n_0000000003) [n:127.0.0.1:59305_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:59305_solr
[junit4] 2> 409496 INFO (jetty-closer-13-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4e60539c{/solr,null,UNAVAILABLE}
[junit4] 2> 409495 WARN (OverseerAutoScalingTriggerThread-99368291327213580-127.0.0.1:59305_solr-n_0000000003) [n:127.0.0.1:59305_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 409503 INFO (jetty-closer-13-thread-2) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 409509 INFO (jetty-closer-13-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@33f3b0b0{/solr,null,UNAVAILABLE}
[junit4] 2> 409510 INFO (jetty-closer-13-thread-1) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 409674 ERROR (SUITE-TestCollectionStateWatchers-seed#[9EDBA3692A4378C6]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 409674 INFO (SUITE-TestCollectionStateWatchers-seed#[9EDBA3692A4378C6]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35127 35127
[junit4] 2> 409801 INFO (Thread-53) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35127 35127
[junit4] 2> 409802 WARN (Thread-53) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/aliases.json
[junit4] 2> 8 /solr/clusterprops.json
[junit4] 2> 7 /solr/security.json
[junit4] 2> 7 /solr/configs/config
[junit4] 2> 4 /solr/collections/stateformat1/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 57 /solr/collections/falsepredicate/state.json
[junit4] 2> 57 /solr/collections/testcollection/state.json
[junit4] 2> 17 /solr/collections/waitforstate/state.json
[junit4] 2> 8 /solr/clusterstate.json
[junit4] 2> 7 /solr/collections/currentstate/state.json
[junit4] 2> 7 /solr/collections/delayed/state.json
[junit4] 2> 6 /solr/collections/tobedeleted/state.json
[junit4] 2> 5 /solr/collections/stateformat1/state.json
[junit4] 2> 3 /solr/overseer_elect/election/99368291327213578-127.0.0.1:36015_solr-n_0000000000
[junit4] 2> 2 /solr/autoscaling.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/live_nodes
[junit4] 2> 8 /solr/collections
[junit4] 2> 2 /solr/overseer/queue
[junit4] 2> 2 /solr/overseer/collection-queue-work
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-solrj/test/J2/temp/solr.common.cloud.TestCollectionStateWatchers_9EDBA3692A4378C6-001
[junit4] 2> Jan 18, 2018 1:45:26 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 6 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1400, maxMBSortInHeap=6.309672053689489, sim=RandomSimilarity(queryNorm=true): {}, locale=zh-CN, timezone=US/Mountain
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=171145704,total=313524224
[junit4] 2> NOTE: All tests run in this JVM: [SolrExampleXMLTest, TestValidatingJsonMap, HyperbolicSineEvaluatorTest, TestSolrJErrorHandling, AddEvaluatorTest, CloudSolrClientBuilderTest, StreamExpressionToExplanationTest, TestCollectionStateWatchers]
[junit4] Completed [35/146 (1!)] on J2 in 131.29s, 9 tests, 1 failure <<< FAILURES!
[...truncated 47567 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 318 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/318/
9 tests failed.
FAILED: org.apache.solr.cloud.TestCloudJSONFacetJoinDomain.testRandom
Error Message:
Error from server at http://127.0.0.1:48892/solr/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection: {"org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection":8}
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:48892/solr/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection: {"org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection":8}
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40:60F8CCF7EF65F933]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
at org.apache.solr.cloud.TestCloudJSONFacetJoinDomain.assertFacetCountsAreCorrect(TestCloudJSONFacetJoinDomain.java:461)
at org.apache.solr.cloud.TestCloudJSONFacetJoinDomain.assertFacetCountsAreCorrect(TestCloudJSONFacetJoinDomain.java:465)
at org.apache.solr.cloud.TestCloudJSONFacetJoinDomain.assertFacetCountsAreCorrect(TestCloudJSONFacetJoinDomain.java:429)
at org.apache.solr.cloud.TestCloudJSONFacetJoinDomain.testRandom(TestCloudJSONFacetJoinDomain.java:370)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitAfterFailedSplit
Error Message:
Shard split did not succeed after a previous failed split attempt left sub-shards in construction state
Stack Trace:
java.lang.AssertionError: Shard split did not succeed after a previous failed split attempt left sub-shards in construction state
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40:EBF97A57627002CA]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:297)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestGenericDistributedQueue.testDistributedQueue
Error Message:
Test abandoned because suite timeout was reached.
Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.sim.TestGenericDistributedQueue
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40]:0)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode
Error Message:
did not finish processing changes
Stack Trace:
java.lang.AssertionError: did not finish processing changes
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40:B55BF45B9148C058]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode(TestLargeCluster.java:297)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testTriggerThrottling
Error Message:
Both triggers should have fired by now
Stack Trace:
java.lang.AssertionError: Both triggers should have fired by now
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40:E99641DD8CAFACD2]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testTriggerThrottling(TestTriggerIntegration.java:171)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testEventFromRestoredState
Error Message:
The trigger did not fire at all
Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40:12825D387708EB2A]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testEventFromRestoredState(TestTriggerIntegration.java:673)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState
Error Message:
Two TriggerAction instances should have been created by now
Stack Trace:
java.lang.AssertionError: Two TriggerAction instances should have been created by now
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40:9A89608764C5AEED]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState(TestTriggerIntegration.java:361)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testCooldown
Error Message:
[CapturedEvent{timestamp=16445112123730749, stage=SUCCEEDED, actionName='null', event={ "id":"3a6cba6973b2c7T746dhdwn6gduim68xvnj3bn16", "source":"node_added_cooldown_trigger", "eventTime":16445096538124999, "eventType":"NODEADDED", "properties":{ "eventTimes":[16445096538124999], "_enqueue_time_":16445109634069299, "nodeNames":["127.0.0.1:10012_solr"]}}, context={}, config={ "trigger":"node_added_cooldown_trigger", "afterAction":[], "stage":[ "FAILED", "SUCCEEDED", "IGNORED"], "class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener", "beforeAction":[]}, message='null'}]
Stack Trace:
java.lang.AssertionError: [CapturedEvent{timestamp=16445112123730749, stage=SUCCEEDED, actionName='null', event={
"id":"3a6cba6973b2c7T746dhdwn6gduim68xvnj3bn16",
"source":"node_added_cooldown_trigger",
"eventTime":16445096538124999,
"eventType":"NODEADDED",
"properties":{
"eventTimes":[16445096538124999],
"_enqueue_time_":16445109634069299,
"nodeNames":["127.0.0.1:10012_solr"]}}, context={}, config={
"trigger":"node_added_cooldown_trigger",
"afterAction":[],
"stage":[
"FAILED",
"SUCCEEDED",
"IGNORED"],
"class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",
"beforeAction":[]}, message='null'}]
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40:230A841C20AF3AB2]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testCooldown(TestTriggerIntegration.java:1100)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12255 lines...]
[junit4] Suite: org.apache.solr.cloud.TestCloudJSONFacetJoinDomain
[junit4] 2> 1693017 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudJSONFacetJoinDomain_12B4E9F85E054F40-001/init-core-data-001
[junit4] 2> 1693018 WARN (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=30 numCloses=30
[junit4] 2> 1693043 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 1693058 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 1693059 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudJSONFacetJoinDomain_12B4E9F85E054F40-001/tempDir-001
[junit4] 2> 1693059 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1693106 INFO (Thread-2750) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1693106 INFO (Thread-2750) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1693198 ERROR (Thread-2750) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1693217 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:55238
[junit4] 2> 1693249 INFO (zkConnectionManagerCallback-1217-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1693359 INFO (jetty-launcher-1214-thread-1) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T11:27:37-10:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 1693360 INFO (jetty-launcher-1214-thread-2) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T11:27:37-10:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 1693360 INFO (jetty-launcher-1214-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1693360 INFO (jetty-launcher-1214-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1693360 INFO (jetty-launcher-1214-thread-1) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 1693360 INFO (jetty-launcher-1214-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@61d95de{/solr,null,AVAILABLE}
[junit4] 2> 1693391 INFO (jetty-launcher-1214-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3d778931{HTTP/1.1,[http/1.1]}{127.0.0.1:48892}
[junit4] 2> 1693391 INFO (jetty-launcher-1214-thread-1) [ ] o.e.j.s.Server Started @1717137ms
[junit4] 2> 1693391 INFO (jetty-launcher-1214-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=48892}
[junit4] 2> 1693391 ERROR (jetty-launcher-1214-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1693392 INFO (jetty-launcher-1214-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 1693392 INFO (jetty-launcher-1214-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1693392 INFO (jetty-launcher-1214-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1693392 INFO (jetty-launcher-1214-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-17T11:50:02.413Z
[junit4] 2> 1693599 INFO (jetty-launcher-1214-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1693599 INFO (jetty-launcher-1214-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1693599 INFO (jetty-launcher-1214-thread-2) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 1693678 INFO (jetty-launcher-1214-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@59346517{/solr,null,AVAILABLE}
[junit4] 2> 1693678 INFO (jetty-launcher-1214-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6e94e9c4{HTTP/1.1,[http/1.1]}{127.0.0.1:35017}
[junit4] 2> 1693678 INFO (jetty-launcher-1214-thread-2) [ ] o.e.j.s.Server Started @1717424ms
[junit4] 2> 1693678 INFO (jetty-launcher-1214-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35017}
[junit4] 2> 1693679 ERROR (jetty-launcher-1214-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1693679 INFO (jetty-launcher-1214-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 1693679 INFO (jetty-launcher-1214-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1693679 INFO (jetty-launcher-1214-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1693679 INFO (jetty-launcher-1214-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-17T11:50:02.700Z
[junit4] 2> 1693828 INFO (zkConnectionManagerCallback-1219-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1693828 INFO (jetty-launcher-1214-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1693870 INFO (zkConnectionManagerCallback-1221-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1693870 INFO (jetty-launcher-1214-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1693926 INFO (jetty-launcher-1214-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55238/solr
[junit4] 2> 1694380 INFO (zkConnectionManagerCallback-1225-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1694398 INFO (jetty-launcher-1214-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55238/solr
[junit4] 2> 1694399 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x16103f45e900003, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1694480 INFO (zkConnectionManagerCallback-1229-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1694481 INFO (zkConnectionManagerCallback-1231-thread-1-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1694698 INFO (zkConnectionManagerCallback-1233-thread-1-processing-n:127.0.0.1:35017_solr) [n:127.0.0.1:35017_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1695360 INFO (jetty-launcher-1214-thread-2) [n:127.0.0.1:35017_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1695360 INFO (jetty-launcher-1214-thread-2) [n:127.0.0.1:35017_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35017_solr
[junit4] 2> 1695361 INFO (jetty-launcher-1214-thread-2) [n:127.0.0.1:35017_solr ] o.a.s.c.Overseer Overseer (id=99365014872260614-127.0.0.1:35017_solr-n_0000000000) starting
[junit4] 2> 1695486 INFO (jetty-launcher-1214-thread-2) [n:127.0.0.1:35017_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35017_solr
[junit4] 2> 1695500 INFO (OverseerStateUpdate-99365014872260614-127.0.0.1:35017_solr-n_0000000000) [n:127.0.0.1:35017_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1695734 INFO (jetty-launcher-1214-thread-1) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1695748 INFO (jetty-launcher-1214-thread-1) [n:127.0.0.1:48892_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1695769 INFO (jetty-launcher-1214-thread-1) [n:127.0.0.1:48892_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48892_solr
[junit4] 2> 1695878 INFO (zkCallback-1232-thread-1-processing-n:127.0.0.1:35017_solr) [n:127.0.0.1:35017_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1696018 INFO (zkCallback-1230-thread-1-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1696717 INFO (jetty-launcher-1214-thread-2) [n:127.0.0.1:35017_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35017.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e7a86c4
[junit4] 2> 1696901 INFO (jetty-launcher-1214-thread-2) [n:127.0.0.1:35017_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35017.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e7a86c4
[junit4] 2> 1696901 INFO (jetty-launcher-1214-thread-2) [n:127.0.0.1:35017_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35017.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e7a86c4
[junit4] 2> 1696910 INFO (jetty-launcher-1214-thread-2) [n:127.0.0.1:35017_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudJSONFacetJoinDomain_12B4E9F85E054F40-001/tempDir-001/node2/.
[junit4] 2> 1697032 INFO (jetty-launcher-1214-thread-1) [n:127.0.0.1:48892_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48892.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e7a86c4
[junit4] 2> 1697191 INFO (jetty-launcher-1214-thread-1) [n:127.0.0.1:48892_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48892.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e7a86c4
[junit4] 2> 1697191 INFO (jetty-launcher-1214-thread-1) [n:127.0.0.1:48892_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48892.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e7a86c4
[junit4] 2> 1697229 INFO (jetty-launcher-1214-thread-1) [n:127.0.0.1:48892_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudJSONFacetJoinDomain_12B4E9F85E054F40-001/tempDir-001/node1/.
[junit4] 2> 1698466 INFO (zkConnectionManagerCallback-1239-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1698550 INFO (zkConnectionManagerCallback-1243-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1698551 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1698553 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55238/solr ready
[junit4] 2> 1700460 INFO (qtp67429917-6510) [n:127.0.0.1:35017_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&property.config=solrconfig-tlog.xml&property.schema=schema_latest.xml&collection.configName=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set&name=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1700468 INFO (OverseerThreadFactory-1838-thread-1-processing-n:127.0.0.1:35017_solr) [n:127.0.0.1:35017_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection
[junit4] 2> 1700698 INFO (OverseerStateUpdate-99365014872260614-127.0.0.1:35017_solr-n_0000000000) [n:127.0.0.1:35017_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:35017/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 1700716 INFO (OverseerStateUpdate-99365014872260614-127.0.0.1:35017_solr-n_0000000000) [n:127.0.0.1:35017_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:48892/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 1703316 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set&newCollection=true&collection=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection&version=2&replicaType=NRT&property.config=solrconfig-tlog.xml&property.schema=schema_latest.xml&coreNodeName=core_node4&name=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3&action=CREATE&numShards=2&shard=shard2&wt=javabin
[junit4] 2> 1703316 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1703468 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set&newCollection=true&collection=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection&version=2&replicaType=NRT&property.config=solrconfig-tlog.xml&property.schema=schema_latest.xml&coreNodeName=core_node2&name=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&wt=javabin
[junit4] 2> 1703468 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1704242 INFO (zkCallback-1230-thread-1-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1704399 INFO (zkCallback-1232-thread-1-processing-n:127.0.0.1:35017_solr) [n:127.0.0.1:35017_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1704402 INFO (zkCallback-1230-thread-1-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1704403 INFO (zkCallback-1230-thread-2-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1704554 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 1704724 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.s.IndexSchema [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] Schema name=example
[junit4] 2> 1706312 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 1706531 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.s.IndexSchema [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] Schema name=example
[junit4] 2> 1706930 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema example/1.6 with uniqueid field id
[junit4] 2> 1707023 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1' using configuration from collection org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection, trusted=true
[junit4] 2> 1707024 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35017.solr.core.org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection.shard1.replica_n1' (registry 'solr.core.org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e7a86c4
[junit4] 2> 1707024 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1707024 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.SolrCore [[org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudJSONFacetJoinDomain_12B4E9F85E054F40-001/tempDir-001/node2/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudJSONFacetJoinDomain_12B4E9F85E054F40-001/tempDir-001/node2/./org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1/data/]
[junit4] 2> 1707130 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=91.4140625, floorSegmentMB=1.3056640625, forceMergeDeletesPctAllowed=9.912107369120065, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.18030994669460867
[junit4] 2> 1707172 WARN (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1707332 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.s.IndexSchema Loaded schema example/1.6 with uniqueid field id
[junit4] 2> 1707599 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.CoreContainer Creating SolrCore 'org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3' using configuration from collection org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection, trusted=true
[junit4] 2> 1707600 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48892.solr.core.org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection.shard2.replica_n3' (registry 'solr.core.org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection.shard2.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e7a86c4
[junit4] 2> 1707600 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1707600 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.SolrCore [[org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudJSONFacetJoinDomain_12B4E9F85E054F40-001/tempDir-001/node1/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudJSONFacetJoinDomain_12B4E9F85E054F40-001/tempDir-001/node1/./org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3/data/]
[junit4] 2> 1707730 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=91.4140625, floorSegmentMB=1.3056640625, forceMergeDeletesPctAllowed=9.912107369120065, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.18030994669460867
[junit4] 2> 1708343 WARN (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1708508 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1708508 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1708515 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1708515 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1708516 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=89.4091796875, floorSegmentMB=1.666015625, forceMergeDeletesPctAllowed=29.841986508731416, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1708538 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2215244e[org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] main]
[junit4] 2> 1708586 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set
[junit4] 2> 1708587 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set
[junit4] 2> 1708587 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1708588 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589840254189174784
[junit4] 2> 1708615 INFO (searcherExecutor-1843-thread-1-processing-n:127.0.0.1:35017_solr x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1 s:shard1 c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection r:core_node2) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.SolrCore [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] Registered new searcher Searcher@2215244e[org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1708617 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1708617 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1708617 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:35017/solr/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1/
[junit4] 2> 1708618 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1708618 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:35017/solr/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 1708618 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1708636 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:35017/solr/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1/ shard1
[junit4] 2> 1708647 INFO (zkCallback-1230-thread-2-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1708647 INFO (zkCallback-1230-thread-1-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1708647 INFO (zkCallback-1232-thread-2-processing-n:127.0.0.1:35017_solr) [n:127.0.0.1:35017_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1708690 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1708711 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set&newCollection=true&collection=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection&version=2&replicaType=NRT&property.config=solrconfig-tlog.xml&property.schema=schema_latest.xml&coreNodeName=core_node2&name=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&wt=javabin} status=0 QTime=5243
[junit4] 2> 1708849 INFO (zkCallback-1232-thread-2-processing-n:127.0.0.1:35017_solr) [n:127.0.0.1:35017_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1708858 INFO (zkCallback-1230-thread-2-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1708858 INFO (zkCallback-1230-thread-1-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1708908 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1708908 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1708909 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1708909 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1708959 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=89.4091796875, floorSegmentMB=1.666015625, forceMergeDeletesPctAllowed=29.841986508731416, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1708996 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.s.SolrIndexSearcher Opening [Searcher@7b452937[org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] main]
[junit4] 2> 1709014 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set
[junit4] 2> 1709015 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set
[junit4] 2> 1709015 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1709017 INFO (searcherExecutor-1844-thread-1-processing-n:127.0.0.1:48892_solr x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3 s:shard2 c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection r:core_node4) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.SolrCore [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] Registered new searcher Searcher@7b452937[org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1709017 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589840254639013888
[junit4] 2> 1709037 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1709037 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1709037 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:48892/solr/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3/
[junit4] 2> 1709037 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1709037 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.SyncStrategy http://127.0.0.1:48892/solr/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3/ has no replicas
[junit4] 2> 1709037 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1709056 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:48892/solr/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3/ shard2
[junit4] 2> 1709249 INFO (zkCallback-1232-thread-2-processing-n:127.0.0.1:35017_solr) [n:127.0.0.1:35017_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1709278 INFO (zkCallback-1230-thread-2-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1709278 INFO (zkCallback-1230-thread-1-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1709450 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1709452 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set&newCollection=true&collection=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection&version=2&replicaType=NRT&property.config=solrconfig-tlog.xml&property.schema=schema_latest.xml&coreNodeName=core_node4&name=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3&action=CREATE&numShards=2&shard=shard2&wt=javabin} status=0 QTime=6135
[junit4] 2> 1709479 INFO (qtp67429917-6510) [n:127.0.0.1:35017_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1709590 INFO (zkCallback-1232-thread-2-processing-n:127.0.0.1:35017_solr) [n:127.0.0.1:35017_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1709598 INFO (zkCallback-1230-thread-1-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1709598 INFO (zkCallback-1230-thread-2-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1710448 INFO (OverseerCollectionConfigSetProcessor-99365014872260614-127.0.0.1:35017_solr-n_0000000000) [n:127.0.0.1:35017_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1710498 INFO (qtp67429917-6510) [n:127.0.0.1:35017_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&property.config=solrconfig-tlog.xml&property.schema=schema_latest.xml&collection.configName=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set&name=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=10038
[junit4] 2> 1710498 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection failOnTimeout:true timeout (sec):330
[junit4] 1> -
[junit4] 1> replica:core_node2 rstate:active live:true
[junit4] 1> replica:core_node4 rstate:active live:true
[junit4] 1> no one is recoverying
[junit4] 2> 1710499 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection
[junit4] 2> 1710535 INFO (qtp67429917-6512) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[0 (1589840256206635008)]} 0 23
[junit4] 2> 1710560 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1 (1589840256254869504)]} 0 1
[junit4] 2> 1710595 INFO (qtp67429917-6512) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1 (1589840256293666816)]} 0 0
[junit4] 2> 1710709 INFO (qtp133797802-6505) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2 (1589840256385941504)]} 0 26
[junit4] 2> 1710814 INFO (qtp133797802-6500) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] webapp=/solr path=/update params={wt=javabin&version=2}{add=[3 (1589840256444661760)]} 0 75
[junit4] 2> 1710816 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[4 (1589840256524353536)]} 0 0
[junit4] 2> 1710817 INFO (qtp67429917-6512) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[4 (1589840256525402112)]} 0 0
[junit4] 2> 1710818 INFO (qtp133797802-6505) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] webapp=/solr path=/update params={wt=javabin&version=2}{add=[5 (1589840256526450688)]} 0 0
[junit4] 2> 1710835 INFO (qtp133797802-6500) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] webapp=/solr path=/update params={wt=javabin&version=2}{add=[6 (1589840256544276480)]} 0 0
[junit4] 2> 1710836 INFO (qtp133797802-6505) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] webapp=/solr path=/update params={wt=javabin&version=2}{add=[7 (1589840256545325056)]} 0 0
[junit4] 2> 1710837 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[8 (1589840256546373632)]} 0 0
[junit4] 2> 1710837 INFO (qtp133797802-6500) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] webapp=/solr path=/update params={wt=javabin&version=2}{add=[9 (1589840256547422208)]} 0 0
[junit4] 2> 1710838 INFO (qtp67429917-6512) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[10 (1589840256548470784)]} 0 0
[junit4] 2> 1710849 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[11 (1589840256560005120)]} 0 0
[junit4] 2> 1710850 INFO (qtp67429917-6512) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[12 (1589840256561053696)]} 0 0
[junit4] 2> 1710860 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[13 (1589840256570490880)]} 0 0
[j
[...truncated too long message...]
ve/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptor-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-admin-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-authn-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-authz-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/bcprov-jdk15on-1.54.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/byte-buddy-1.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/commons-math3-3.6.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/mockito-core-2.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/objenesis-2.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/analysis/icu/lucene-analyzers-icu-7.3.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/backward-codecs/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/contrib/analysis-extras/lib/icu4j-60.2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.4.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_144/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20180117_112125_0264535385095341195582.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20180117_112125_0264849570380174238636.suites -stdin
[junit4] ERROR: JVM J2 ended with an exception: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20180117_112125_0262319053265880513310.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20180117_112125_0262637713925886892793.syserr
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1519)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
[junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] at java.lang.Thread.run(Thread.java:748)
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:836: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:780: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build.xml:262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/common-build.xml:556: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1513: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1038: At least one slave process threw an exception, first: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20180117_112125_0262319053265880513310.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20180117_112125_0262637713925886892793.syserr
Total time: 443 minutes 4 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-Tests-7.x #317 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 317 - Still unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/317/
1 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.testReadApi
Error Message:
expected:<1> but was:<0>
Stack Trace:
java.lang.AssertionError: expected:<1> but was:<0>
at __randomizedtesting.SeedInfo.seed([75C131060C852E87:22E8CAB3D777CC9C]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.testReadApi(AutoScalingHandlerTest.java:725)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12659 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/init-core-data-001
[junit4] 2> 2193689 WARN (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=9 numCloses=9
[junit4] 2> 2193710 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 2193711 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 2193712 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 2193712 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001
[junit4] 2> 2193712 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2193712 INFO (Thread-1839) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2193712 INFO (Thread-1839) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2193752 ERROR (Thread-1839) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2193819 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:35315
[junit4] 2> 2194046 INFO (zkConnectionManagerCallback-2441-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2194170 INFO (jetty-launcher-2438-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2194391 INFO (jetty-launcher-2438-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@44a77e0{/solr,null,AVAILABLE}
[junit4] 2> 2194392 INFO (jetty-launcher-2438-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5b418e5a{SSL,[ssl, http/1.1]}{127.0.0.1:41864}
[junit4] 2> 2194392 INFO (jetty-launcher-2438-thread-1) [ ] o.e.j.s.Server Started @2203742ms
[junit4] 2> 2194392 INFO (jetty-launcher-2438-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41864}
[junit4] 2> 2194392 ERROR (jetty-launcher-2438-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2194392 INFO (jetty-launcher-2438-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 2194392 INFO (jetty-launcher-2438-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2194392 INFO (jetty-launcher-2438-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2194392 INFO (jetty-launcher-2438-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-16T21:23:03.571Z
[junit4] 2> 2194436 INFO (jetty-launcher-2438-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2194496 INFO (jetty-launcher-2438-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3375abbd{/solr,null,AVAILABLE}
[junit4] 2> 2194496 INFO (jetty-launcher-2438-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5d3f40ce{SSL,[ssl, http/1.1]}{127.0.0.1:43663}
[junit4] 2> 2194496 INFO (jetty-launcher-2438-thread-2) [ ] o.e.j.s.Server Started @2203847ms
[junit4] 2> 2194496 INFO (jetty-launcher-2438-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43663}
[junit4] 2> 2194497 ERROR (jetty-launcher-2438-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2194497 INFO (jetty-launcher-2438-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 2194497 INFO (jetty-launcher-2438-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2194497 INFO (jetty-launcher-2438-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2194497 INFO (jetty-launcher-2438-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-16T21:23:03.676Z
[junit4] 2> 2194544 INFO (zkConnectionManagerCallback-2443-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2194544 INFO (zkConnectionManagerCallback-2445-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2194553 INFO (jetty-launcher-2438-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2194553 INFO (jetty-launcher-2438-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2194743 INFO (jetty-launcher-2438-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35315/solr
[junit4] 2> 2194799 INFO (zkConnectionManagerCallback-2449-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2194824 INFO (jetty-launcher-2438-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35315/solr
[junit4] 2> 2194852 INFO (zkConnectionManagerCallback-2453-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2194985 INFO (zkConnectionManagerCallback-2455-thread-1-processing-n:127.0.0.1:41864_solr) [n:127.0.0.1:41864_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2194997 INFO (zkConnectionManagerCallback-2457-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2195536 INFO (jetty-launcher-2438-thread-2) [n:127.0.0.1:43663_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2195537 INFO (jetty-launcher-2438-thread-2) [n:127.0.0.1:43663_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43663_solr
[junit4] 2> 2195537 INFO (jetty-launcher-2438-thread-2) [n:127.0.0.1:43663_solr ] o.a.s.c.Overseer Overseer (id=99361605739610118-127.0.0.1:43663_solr-n_0000000000) starting
[junit4] 2> 2195601 INFO (jetty-launcher-2438-thread-2) [n:127.0.0.1:43663_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43663_solr
[junit4] 2> 2195873 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2196041 INFO (jetty-launcher-2438-thread-2) [n:127.0.0.1:43663_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43663.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2196078 INFO (jetty-launcher-2438-thread-2) [n:127.0.0.1:43663_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43663.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2196091 INFO (jetty-launcher-2438-thread-2) [n:127.0.0.1:43663_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43663.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2196092 INFO (jetty-launcher-2438-thread-2) [n:127.0.0.1:43663_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/.
[junit4] 2> 2196152 INFO (jetty-launcher-2438-thread-1) [n:127.0.0.1:41864_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2196153 INFO (jetty-launcher-2438-thread-1) [n:127.0.0.1:41864_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2196155 INFO (jetty-launcher-2438-thread-1) [n:127.0.0.1:41864_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41864_solr
[junit4] 2> 2196168 INFO (zkCallback-2456-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2196274 INFO (zkCallback-2454-thread-1-processing-n:127.0.0.1:41864_solr) [n:127.0.0.1:41864_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2196522 INFO (jetty-launcher-2438-thread-1) [n:127.0.0.1:41864_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41864.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2196574 INFO (jetty-launcher-2438-thread-1) [n:127.0.0.1:41864_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41864.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2196574 INFO (jetty-launcher-2438-thread-1) [n:127.0.0.1:41864_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41864.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2196575 INFO (jetty-launcher-2438-thread-1) [n:127.0.0.1:41864_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node1/.
[junit4] 2> 2196732 INFO (zkConnectionManagerCallback-2463-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2196734 INFO (zkConnectionManagerCallback-2467-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2196735 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2196751 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35315/solr ready
[junit4] 2> 2196859 INFO (TEST-AutoScalingHandlerTest.testDeleteUsedPolicy-seed#[75C131060C852E87]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDeleteUsedPolicy
[junit4] 2> 2197223 INFO (qtp1964390929-10760) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=2
[junit4] 2> 2197268 INFO (qtp2066369429-10772) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 2197282 INFO (qtp2066369429-10770) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=197
[junit4] 2> 2197284 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=COLL1&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2&policy=nodelete and sendToOCPQueue=true
[junit4] 2> 2197286 INFO (OverseerThreadFactory-3840-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.CreateCollectionCmd Create collection COLL1
[junit4] 2> 2197346 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=3
[junit4] 2> 2197404 INFO (qtp2066369429-10765) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2197426 INFO (OverseerThreadFactory-3840-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 592118598 sessionWrapper.createTime 592118577050542, this.sessionWrapper.createTime 592118577050542
[junit4] 2> 2197586 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"COLL1",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"COLL1_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:43663/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2197815 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=COLL1_shard1_replica_n1&action=CREATE&numShards=1&collection=COLL1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2197815 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2197959 INFO (zkCallback-2456-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/COLL1/state.json] for collection [COLL1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2197983 INFO (zkCallback-2456-thread-2-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/COLL1/state.json] for collection [COLL1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2198945 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2198989 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.s.IndexSchema [COLL1_shard1_replica_n1] Schema name=minimal
[junit4] 2> 2198991 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2198991 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'COLL1_shard1_replica_n1' using configuration from collection COLL1, trusted=true
[junit4] 2> 2199008 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43663.solr.core.COLL1.shard1.replica_n1' (registry 'solr.core.COLL1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2199008 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2199008 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.SolrCore [[COLL1_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/COLL1_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/./COLL1_shard1_replica_n1/data/]
[junit4] 2> 2199257 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2199257 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2199258 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2199258 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2199272 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6ad16831[COLL1_shard1_replica_n1] main]
[junit4] 2> 2199304 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2199304 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2199304 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2199305 INFO (searcherExecutor-3845-thread-1-processing-n:127.0.0.1:43663_solr x:COLL1_shard1_replica_n1 s:shard1 c:COLL1 r:core_node2) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.SolrCore [COLL1_shard1_replica_n1] Registered new searcher Searcher@6ad16831[COLL1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2199305 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589785697697398784
[junit4] 2> 2199381 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2199381 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2199381 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:43663/solr/COLL1_shard1_replica_n1/
[junit4] 2> 2199381 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2199381 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:43663/solr/COLL1_shard1_replica_n1/ has no replicas
[junit4] 2> 2199381 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2199401 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:43663/solr/COLL1_shard1_replica_n1/ shard1
[junit4] 2> 2199575 INFO (zkCallback-2456-thread-2-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/COLL1/state.json] for collection [COLL1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2199575 INFO (zkCallback-2456-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/COLL1/state.json] for collection [COLL1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2199611 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2199613 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=COLL1_shard1_replica_n1&action=CREATE&numShards=1&collection=COLL1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1798
[junit4] 2> 2199629 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2199733 INFO (zkCallback-2456-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/COLL1/state.json] for collection [COLL1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2199733 INFO (zkCallback-2456-thread-2-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/COLL1/state.json] for collection [COLL1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2200629 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=COLL1&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2&policy=nodelete} status=0 QTime=3345
[junit4] 2> 2200633 ERROR (qtp2066369429-10769) [n:127.0.0.1:43663_solr ] o.a.s.h.RequestHandlerBase org.apache.solr.api.ApiBag$ExceptionWithErrObject: Error in command payload, errors: [{remove-policy=nodelete, errorMessages=[policy : nodelete is being used by collection COLL1]}],
[junit4] 2> at org.apache.solr.cloud.autoscaling.AutoScalingHandler.processOps(AutoScalingHandler.java:207)
[junit4] 2> at org.apache.solr.cloud.autoscaling.AutoScalingHandler.handleRequestBody(AutoScalingHandler.java:145)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
[junit4] 2> at org.apache.solr.api.ApiBag$ReqHandlerToApi.call(ApiBag.java:242)
[junit4] 2> at org.apache.solr.api.V2HttpCall.handleAdmin(V2HttpCall.java:311)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2>
[junit4] 2> 2200633 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=400 QTime=1
[junit4] 2> 2200634 INFO (qtp2066369429-10772) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=COLL1&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2200640 INFO (OverseerCollectionConfigSetProcessor-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2200641 INFO (OverseerThreadFactory-3840-thread-2-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
[junit4] 2> 2200641 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.COLL1.shard1.replica_n1, tag=null
[junit4] 2> 2200641 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1352bac3: rootName = solr_43663, domain = solr.core.COLL1.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.COLL1.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@625e23b9
[junit4] 2> 2200684 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.c.SolrCore [COLL1_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@7db03a62
[junit4] 2> 2200684 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.COLL1.shard1.replica_n1, tag=2108701282
[junit4] 2> 2200684 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.COLL1.shard1.leader, tag=2108701282
[junit4] 2> 2200707 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=COLL1_shard1_replica_n1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=65
[junit4] 2> 2201441 INFO (qtp2066369429-10772) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=COLL1&action=DELETE&wt=javabin&version=2} status=0 QTime=806
[junit4] 2> 2201441 INFO (TEST-AutoScalingHandlerTest.testDeleteUsedPolicy-seed#[75C131060C852E87]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDeleteUsedPolicy
[junit4] 2> 2201465 INFO (TEST-AutoScalingHandlerTest.testErrorHandling-seed#[75C131060C852E87]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testErrorHandling
[junit4] 2> 2201467 ERROR (qtp2066369429-10770) [n:127.0.0.1:43663_solr ] o.a.s.h.RequestHandlerBase org.apache.solr.api.ApiBag$ExceptionWithErrObject: Error in command payload, errors: [{set-cluster-policy=[{cores=<10, node=#ANY}, {shard=#EACH, node=#ANY}, {nodeRole=overseer, replica=0}], errorMessages=[replica is required in {
[junit4] 2> "shard":"#EACH",
[junit4] 2> "node":"#ANY"}]}],
[junit4] 2> at org.apache.solr.cloud.autoscaling.AutoScalingHandler.processOps(AutoScalingHandler.java:207)
[junit4] 2> at org.apache.solr.cloud.autoscaling.AutoScalingHandler.handleRequestBody(AutoScalingHandler.java:145)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2>
[junit4] 2> 2201468 INFO (qtp2066369429-10770) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=400 QTime=2
[junit4] 2> 2201468 INFO (TEST-AutoScalingHandlerTest.testErrorHandling-seed#[75C131060C852E87]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testErrorHandling
[junit4] 2> 2201496 INFO (TEST-AutoScalingHandlerTest.testReadApi-seed#[75C131060C852E87]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testReadApi
[junit4] 2> 2201514 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201528 INFO (qtp1964390929-10759) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=12
[junit4] 2> 2201580 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=68
[junit4] 2> 2201605 INFO (qtp2066369429-10772) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 2201606 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 2201607 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=12
[junit4] 2> 2201616 INFO (qtp2066369429-10772) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=1
[junit4] 2> 2201617 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201618 INFO (qtp1964390929-10757) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=10
[junit4] 2> 2201620 INFO (qtp2066369429-10765) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201633 INFO (qtp1964390929-10760) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=12
[junit4] 2> 2201637 INFO (qtp2066369429-10770) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=18
[junit4] 2> 2201639 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 2201654 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201657 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201670 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling/diagnostics params={wt=javabin&version=2} status=0 QTime=29
[junit4] 2> 2201670 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=6&collection.configName=conf&maxShardsPerNode=10&name=readApiTestViolations&nrtReplicas=6&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2201676 INFO (OverseerThreadFactory-3840-thread-3-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.CreateCollectionCmd Create collection readApiTestViolations
[junit4] 2> 2201676 INFO (OverseerCollectionConfigSetProcessor-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2201677 ERROR (OverseerThreadFactory-3840-thread-3-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.OverseerCollectionMessageHandler Collection: readApiTestViolations operation: create failed:org.apache.solr.common.SolrException: 'maxShardsPerNode>0' is not supported when autoScaling policies are used
[junit4] 2> at org.apache.solr.cloud.CreateCollectionCmd.buildReplicaPositions(CreateCollectionCmd.java:294)
[junit4] 2> at org.apache.solr.cloud.CreateCollectionCmd.call(CreateCollectionCmd.java:123)
[junit4] 2> at org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:243)
[junit4] 2> at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:469)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2>
[junit4] 2> 2201698 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=6&collection.configName=conf&maxShardsPerNode=10&name=readApiTestViolations&nrtReplicas=6&action=CREATE&numShards=1&wt=javabin&version=2} status=400 QTime=28
[junit4] 2> 2201701 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201710 INFO (qtp1964390929-10759) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201711 INFO (qtp2066369429-10772) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=11
[junit4] 2> 2201712 INFO (qtp2066369429-10765) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=6&collection.configName=conf&name=readApiTestViolations&nrtReplicas=6&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2201720 INFO (OverseerThreadFactory-3840-thread-4-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.CreateCollectionCmd Create collection readApiTestViolations
[junit4] 2> 2201720 INFO (OverseerCollectionConfigSetProcessor-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2201721 WARN (OverseerThreadFactory-3840-thread-4-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 6 on collection readApiTestViolations is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
[junit4] 2> 2201726 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201743 INFO (qtp1964390929-10755) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201766 INFO (OverseerThreadFactory-3840-thread-4-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 592122938 sessionWrapper.createTime 592122915211723, this.sessionWrapper.createTime 592122915211723
[junit4] 2> 2201995 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"readApiTestViolations",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"readApiTestViolations_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:43663/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2201997 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"readApiTestViolations",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"readApiTestViolations_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:41864/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2202006 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"readApiTestViolations",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"readApiTestViolations_shard1_replica_n4",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:43663/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2202008 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"readApiTestViolations",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"readApiTestViolations_shard1_replica_n6",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:41864/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2202009 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"readApiTestViolations",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"readApiTestViolations_shard1_replica_n8",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:43663/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2202027 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"readApiTestViolations",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"readApiTestViolations_shard1_replica_n10",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:41864/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2202228 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=readApiTestViolations_shard1_replica_n1&action=CREATE&numShards=1&collection=readApiTestViolations&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2202230 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=readApiTestViolations_shard1_replica_n2&action=CREATE&numShards=1&collection=readApiTestViolations&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2202230 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2202232 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=readApiTestViolations_shard1_replica_n4&action=CREATE&numShards=1&collection=readApiTestViolations&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2202233 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node9&collection.configName=conf&newCollection=true&name=readApiTestViolations_shard1_replica_n6&action=CREATE&numShards=1&collection=readApiTestViolations&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2202233 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node11&collection.configName=conf&newCollection=true&name=readApiTestViolations_shard1_replica_n8&action=CREATE&numShards=1&collection=readApiTestViolations&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2202237 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node12&collection.configName=conf&newCollection=true&name=readApiTestViolations_shard1_replica_n10&action=CREATE&numShards=1&collection=readApiTestViolations&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2202370 INFO (zkCallback-2454-thread-1-processing-n:127.0.0.1:41864_solr) [n:127.0.0.1:41864_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2202370 INFO (zkCallback-2456-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2202371 INFO (zkCallback-2454-thread-2-processing-n:127.0.0.1:41864_solr) [n:127.0.0.1:41864_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2202386 INFO (zkCallback-2456-thread-3-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2203376 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2203392 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2203410 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2203449 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.s.IndexSchema [readApiTestViolations_shard1_replica_n10] Schema name=minimal
[junit4] 2> 2203449 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2203451 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2203451 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.c.CoreContainer Creating SolrCore 'readApiTestViolations_shard1_replica_n10' using configuration from collection readApiTestViolations, trusted=true
[junit4] 2> 2203465 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.s.IndexSchema [readApiTestViolations_shard1_replica_n4] Schema name=minimal
[junit4] 2> 2203467 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2203467 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'readApiTestViolations_shard1_replica_n4' using configuration from collection readApiTestViolations, trusted=true
[junit4] 2> 2203469 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.s.IndexSchema [readApiTestViolations_shard1_replica_n6] Schema name=minimal
[junit4] 2> 2203470 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2203471 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.c.CoreContainer Creating SolrCore 'readApiTestViolations_shard1_replica_n6' using configuration from collection readApiTestViolations, trusted=true
[junit4] 2> 2203472 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41864.solr.core.readApiTestViolations.shard1.replica_n10' (registry 'solr.core.readApiTestViolations.shard1.replica_n10') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2203472 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2203472 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.c.SolrCore [[readApiTestViolations_shard1_replica_n10] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node1/readApiTestViolations_shard1_replica_n10], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node1/./readApiTestViolations_shard1_replica_n10/data/]
[junit4] 2> 2203477 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43663.solr.core.readApiTestViolations.shard1.replica_n4' (registry 'solr.core.readApiTestViolations.shard1.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2203477 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2203477 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.SolrCore [[readApiTestViolations_shard1_replica_n4] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/readApiTestViolations_shard1_replica_n4], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/./readApiTestViolations_shard1_replica_n4/data/]
[junit4] 2> 2203492 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41864.solr.core.readApiTestViolations.shard1.replica_n6' (registry 'solr.core.readApiTestViolations.shard1.replica_n6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2203492 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2203492 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.c.SolrCore [[readApiTestViolations_shard1_replica_n6] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node1/readApiTestViolations_shard1_replica_n6], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node1/./readApiTestViolations_shard1_replica_n6/data/]
[junit4] 2> 2203506 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2203701 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2203749 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.s.IndexSchema [readApiTestViolations_shard1_replica_n2] Schema name=minimal
[junit4] 2> 2203750 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.s.IndexSchema [readApiTestViolations_shard1_replica_n1] Schema name=minimal
[junit4] 2> 2203751 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2203751 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'readApiTestViolations_shard1_replica_n2' using configuration from collection readApiTestViolations, trusted=true
[junit4] 2> 2203764 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41864.solr.core.readApiTestViolations.shard1.replica_n2' (registry 'solr.core.readApiTestViolations.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2203764 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2203764 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.c.SolrCore [[readApiTestViolations_shard1_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node1/readApiTestViolations_shard1_replica_n2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node1/./readApiTestViolations_shard1_replica_n2/data/]
[junit4] 2> 2203817 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.s.IndexSchema [readApiTestViolations_shard1_replica_n8] Schema name=minimal
[junit4] 2> 2203819 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2203819 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.c.CoreContainer Creating SolrCore 'readApiTestViolations_shard1_replica_n8' using configuration from collection readApiTestViolations, trusted=true
[junit4] 2> 2203820 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2203820 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'readApiTestViolations_shard1_replica_n1' using configuration from collection readApiTestViolations, trusted=true
[junit4] 2> 2203832 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43663.solr.core.readApiTestViolations.shard1.replica_n1' (registry 'solr.core.readApiTestViolations.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2203832 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2203832 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.c.SolrCore [[readApiTestViolations_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/readApiTestViolations_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/./readApiTestViolations_shard1_replica_n1/data/]
[junit4] 2> 2203869 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43663.solr.core.readApiTestViolations.shard1.replica_n8' (registry 'solr.core.readApiTestViolations.shard1.replica_n8') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2203869 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2203869 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.c.SolrCore [[readApiTestViolations_shard1_replica_n8] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/readApiTestViolations_shard1_replica_n8], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/./readApiTestViolations_shard1_replica_n8/data/]
[junit4] 2> 2204286 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2204286 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2204287 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2204287 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2204288 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@39c8a774[readApiTestViolations_shard1_replica_n4] main]
[junit4] 2> 2204343 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2204359 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2204371 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2204371 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2204371 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2204372 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589785703010533376
[junit4] 2> 2204372 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2204372 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2204373 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2204373 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2204374 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2204374 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2204375 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=6 found=1 timeoutin=9999ms
[junit4] 2> 2204380 INFO (searcherExecutor-3851-thread-1-processing-n:127.0.0.1:43663_solr x:readApiTestViolations_shard1_replica_n4 s:shard1 c:readApiTestViolations r:core_node7) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.SolrCore [readApiTestViolations_shard1_replica_n4] Registered new searcher Searcher@39c8a774[readApiTestViolations_shard1_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2204380 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.s.SolrIndexSearcher Opening [Searcher@25b2c56c[readApiTestViolations_shard1_replica_n10] main]
[junit4] 2> 2204400 INFO (zkCallback-2456-thread-2-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2204400 INFO (zkCallback-2456-thread-3-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2204404 INFO (zkCallback-2454-thread-2-processing-n:127.0.0.1:41864_solr) [n:127.0.0.1:41864_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2204404 INFO (zkCallback-2454-thread-1-processing-n:127.0.0.1:41864_solr) [n:127.0.0.1:41864_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2204416 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2204416 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2204417 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2204417 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2204418 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2204418 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2204419 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2204419 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2204428 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.s.SolrIndexSearcher Opening [Searcher@31642988[readApiTestViolations_shard1_replica_n6] main]
[junit4] 2> 2204436 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@619a949e[readApiTestViolations_shard1_replica_n1] main]
[junit4] 2> 2204437 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.s.SolrIndexSearcher Opening [Searcher@14337e68[readApiTestViolations_shard1_replica_n8] main]
[junit4] 2> 2204444 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2204444 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2204445 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2204445 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2204446 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@622bb469[readApiTestViolations_shard1_replica_n2] main]
[junit4] 2> 2204453 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2204453 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2204453 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2204453 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2204454 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589785703096516608
[junit4] 2> 2204456 INFO (searcherExecutor-3855-thread-1-processing-n:127.0.0.1:43663_solr x:readApiTestViolations_shard1_replica_n1 s:shard1 c:readApiTestViolations r:core_node3) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.c.SolrCore [readApiTestViolations_shard1_replica_n1] Registered new searcher Searcher@619a949e[readApiTestViolations_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2204456 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2204457 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2204457 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589785703099662336
[junit4] 2> 2204459 INFO (searcherExecutor-3854-thread-1-processing-n:127.0.0.1:43663_solr x:readApiTestViolations_shard1_replica_n8 s:shard1 c:readApiTestViolations r:core_node11) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.c.SolrCore [readApiTestViolations_shard1_replica_n8] Registered new searcher Searcher@14337e68[readApiTestViolations_shard1_replica_n8] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2204468 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2204468 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2204469 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2204469 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2204470 INFO (searcherExecutor-3850-thread-1-processing-n:127.0.0.1:41864_solr x:readApiTestViolations_shard1_replica_n10 s:shard1 c:readApiTestViolations r:core_node12) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.c.SolrCore [readApiTestViolations_shard1_replica_n10] Registered new searcher Searcher@25b2c56c[readApiTestViolations_shard1_replica_n10] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2204470 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2204470 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589785703113293824
[junit4] 2> 2204471 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2204472 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2204472 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2204606 INFO (searcherExecutor-3852-thread-1-processing-n:127.0.0.1:41864_solr x:readApiTestViolations_shard1_replica_n6 s:shard1 c:readApiTestViolations r:core_node9) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.c.SolrCore [readApiTestViolations_shard1_replica_n6] Registered new searcher Searcher@31642988[readApiTestViolations_shard1_replica_n6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2204606 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2204607 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589785703256948736
[junit4] 2> 2204609 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589785703259045888
[junit4] 2> 2204620 INFO (searcherExecutor-3853-thread-1-processing-n:127.0.0.1:41864_solr x:readApiTestViolations_shard1_replica_n2 s:shard1 c:readApiTestViolations r:core_node5) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.c.SolrCore [readApiTestViolations_shard1_replica_n2] Registered new searcher Searcher@622bb469[readApiTestViolations_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2204876 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2204876 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2204876 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:43663/solr/readApiTestViolations_shard1_replica_n4/
[junit4] 2> 2204876 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.u.PeerSync PeerSync: core=readApiTestViolations_shard1_replica_n4 url=https://127.0.0.1:43663/solr START replicas=[https://127.0.0.1:43663/solr/readApiTestViolations_shard1_replica_n1/, https://127.0.0.1:41864/solr/readApiTestViolations_shard1_replica_n2/, https://127.0.0.1:41864/solr/readApiTestViolations_shard1_replica_n6/, https://127.0.0.1:43663/solr/readApiTestViolations_shard1_replica_n8/, https://127.0.0.1:41864/solr/readApiTestViolations_shard
[...truncated too long message...]
ApiTestViolations] has occurred - updating... (live nodes size: [0])
[junit4] 2> 2213370 INFO (coreCloseExecutor-3888-thread-2) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.c.SolrCore [readApiTestViolations_shard1_replica_n2] CLOSING SolrCore org.apache.solr.core.SolrCore@23932760
[junit4] 2> 2213370 INFO (coreCloseExecutor-3888-thread-2) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.readApiTestViolations.shard1.replica_n2, tag=596846432
[junit4] 2> 2213370 INFO (coreCloseExecutor-3888-thread-2) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@15a5ebb3: rootName = solr_41864, domain = solr.core.readApiTestViolations.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.readApiTestViolations.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@36bb95a6
[junit4] 2> 2213370 INFO (zkCallback-2454-thread-3-processing-n:127.0.0.1:41864_solr) [n:127.0.0.1:41864_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [0])
[junit4] 2> 2213371 INFO (jetty-closer-2439-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@27e49c02{/solr,null,UNAVAILABLE}
[junit4] 2> 2213380 INFO (zkCallback-2456-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [0])
[junit4] 2> 2213406 INFO (coreCloseExecutor-3886-thread-2) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.readApiTestViolations.shard1.leader, tag=1860750354
[junit4] 2> 2213406 INFO (coreCloseExecutor-3886-thread-1) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.readApiTestViolations.shard1.leader, tag=541773479
[junit4] 2> 2213406 INFO (coreCloseExecutor-3886-thread-3) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.readApiTestViolations.shard1.replica_n8, tag=651912070
[junit4] 2> 2213406 INFO (coreCloseExecutor-3886-thread-3) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@33aa9b5c: rootName = solr_43663, domain = solr.core.readApiTestViolations.shard1.replica_n8, service url = null, agent id = null] for registry solr.core.readApiTestViolations.shard1.replica_n8 / com.codahale.metrics.MetricRegistry@4cc67e32
[junit4] 2> 2213440 INFO (coreCloseExecutor-3888-thread-2) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.readApiTestViolations.shard1.leader, tag=596846432
[junit4] 2> 2213474 INFO (coreCloseExecutor-3886-thread-3) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.readApiTestViolations.shard1.leader, tag=651912070
[junit4] 2> 2213535 INFO (jetty-closer-2439-thread-1) [ ] o.a.s.c.Overseer Overseer (id=99361605739610118-127.0.0.1:43663_solr-n_0000000000) closing
[junit4] 2> 2213551 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:43663_solr
[junit4] 2> 2213583 WARN (OverseerAutoScalingTriggerThread-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 2213676 INFO (jetty-closer-2439-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@44a77e0{/solr,null,UNAVAILABLE}
[junit4] 2> 2215171 INFO (jetty-closer-2439-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3375abbd{/solr,null,UNAVAILABLE}
[junit4] 2> 2215172 ERROR (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2215172 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35315 35315
[junit4] 2> 2215483 INFO (Thread-1839) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35315 35315
[junit4] 2> 2215483 WARN (Thread-1839) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/aliases.json
[junit4] 2> 3 /solr/security.json
[junit4] 2> 3 /solr/configs/conf
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 18 /solr/collections/readApiTestViolations/state.json
[junit4] 2> 5 /solr/collections/COLL1/state.json
[junit4] 2> 4 /solr/clusterprops.json
[junit4] 2> 4 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/live_nodes
[junit4] 2> 4 /solr/collections
[junit4] 2> 3 /solr/autoscaling/events/node_lost_trigger
[junit4] 2> 2 /solr/autoscaling/events/node_added_trigger
[junit4] 2> 2 /solr/autoscaling/events/node_added_trigger1
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001
[junit4] 2> Jan 16, 2018 9:23:24 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=747, maxMBSortInHeap=6.8184077987319505, sim=RandomSimilarity(queryNorm=true): {}, locale=fi, timezone=Asia/Shanghai
[junit4] 2> NOTE: Linux 4.4.0-104-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=294681568,total=522190848
[junit4] 2> NOTE: All tests run in this JVM: [MetricsConfigTest, TestCloudRecovery, TestHalfAndHalfDocValues, TestJmxIntegration, TestNumericTerms64, DistributedVersionInfoTest, TestExecutePlanAction, TestManagedSynonymFilterFactory, HdfsBasicDistributedZk2Test, TestReloadAndDeleteDocs, TestOmitPositions, TestSubQueryTransformer, TestMultiWordSynonyms, TlogReplayBufferedWhileIndexingTest, TestQuerySenderNoQuery, BooleanFieldTest, ZkSolrClientTest, TestRestoreCore, ZkFailoverTest, TestExpandComponent, TestTolerantUpdateProcessorCloud, CollectionTooManyReplicasTest, TestStressInPlaceUpdates, TestHdfsBackupRestoreCore, ExitableDirectoryReaderTest, SuggestComponentContextFilterQueryTest, SimpleMLTQParserTest, ShardSplitTest, TestRandomCollapseQParserPlugin, TestBinaryResponseWriter, TestManagedResource, CircularListTest, ImplicitSnitchTest, TestManagedSchemaThreadSafety, TestSurroundQueryParser, TestSolrCoreSnapshots, TestLegacyNumericRangeQueryBuilder, ConvertedLegacyTest, TestCustomSort, WrapperMergePolicyFactoryTest, TestSizeLimitedDistributedMap, TestRTimerTree, PolyFieldTest, TestBulkSchemaAPI, TestApiFramework, TestSolrConfigHandlerCloud, OutputWriterTest, TestSolrCloudWithKerberosAlt, TestConfigSetsAPIExclusivity, PeerSyncWithIndexFingerprintCachingTest, ReplaceNodeTest, OverriddenZkACLAndCredentialsProvidersTest, ConjunctionSolrSpellCheckerTest, TestCustomStream, TestSolr4Spatial, SolrGraphiteReporterTest, DOMUtilTest, TestStressUserVersions, SolrMetricReporterTest, OpenCloseCoreStressTest, SolrIndexConfigTest, NumericFieldsTest, ShardRoutingCustomTest, TestLockTree, BufferStoreTest, SynonymTokenizerTest, TestIntervalFaceting, TestSimDistributedQueue, DirectUpdateHandlerTest, TestCloudJSONFacetJoinDomain, TestDefaultStatsCache, TestLuceneMatchVersion, AddSchemaFieldsUpdateProcessorFactoryTest, BigEndianAscendingWordSerializerTest, TestGeoJSONResponseWriter, LukeRequestHandlerTest, TestFastWriter, BasicAuthStandaloneTest, SampleTest, ConfigureRecoveryStrategyTest, TestInfoStreamLogging, TestFastLRUCache, TestHighFrequencyDictionaryFactory, SolrCmdDistributorTest, PreAnalyzedFieldTest, TestCollationField, ReplicaListTransformerTest, CdcrBidirectionalTest, TestAnalyzedSuggestions, TestInPlaceUpdatesStandalone, TestRangeQuery, TestDelegationWithHadoopAuth, TolerantUpdateProcessorTest, TestAuthenticationFramework, SuggesterTSTTest, TestSimpleTrackingShardHandler, HdfsUnloadDistributedZkTest, OpenExchangeRatesOrgProviderTest, AnalyticsMergeStrategyTest, TestDocTermOrds, AutoScalingHandlerTest]
[junit4] Completed [288/767 (1!)] on J1 in 22.34s, 8 tests, 1 failure <<< FAILURES!
[...truncated 49783 lines...]