You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2017/03/20 04:12:25 UTC
[JENKINS] Lucene-Solr-Tests-master - Build # 1750 - Unstable
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/1750/
2 tests failed.
FAILED: org.apache.solr.cloud.OverseerRolesTest.testOverseerRole
Error Message:
Timed out waiting for overseer state change
Stack Trace:
java.lang.AssertionError: Timed out waiting for overseer state change
at __randomizedtesting.SeedInfo.seed([EBA0DE45B66DBB6:EF71F07060D5ED67]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.OverseerRolesTest.waitForNewOverseer(OverseerRolesTest.java:62)
at org.apache.solr.cloud.OverseerRolesTest.testOverseerRole(OverseerRolesTest.java:140)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
FAILED: org.apache.solr.cloud.ShardSplitTest.testSplitWithChaosMonkey
Error Message:
There are still nodes recoverying - waited for 330 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
at __randomizedtesting.SeedInfo.seed([EBA0DE45B66DBB6:859DDE351A607032]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:187)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:144)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:870)
at org.apache.solr.cloud.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:442)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12486 lines...]
[junit4] Suite: org.apache.solr.cloud.OverseerRolesTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.OverseerRolesTest_EBA0DE45B66DBB6-001/init-core-data-001
[junit4] 2> 1337694 WARN (SUITE-OverseerRolesTest-seed#[EBA0DE45B66DBB6]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=6 numCloses=6
[junit4] 2> 1337694 INFO (SUITE-OverseerRolesTest-seed#[EBA0DE45B66DBB6]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields
[junit4] 2> 1337695 INFO (SUITE-OverseerRolesTest-seed#[EBA0DE45B66DBB6]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 1337695 INFO (SUITE-OverseerRolesTest-seed#[EBA0DE45B66DBB6]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.OverseerRolesTest_EBA0DE45B66DBB6-001/tempDir-001
[junit4] 2> 1337695 INFO (SUITE-OverseerRolesTest-seed#[EBA0DE45B66DBB6]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1337696 INFO (Thread-5101) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1337696 INFO (Thread-5101) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1337796 INFO (SUITE-OverseerRolesTest-seed#[EBA0DE45B66DBB6]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:49006
[junit4] 2> 1337803 INFO (jetty-launcher-2020-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1337803 INFO (jetty-launcher-2020-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1337805 INFO (jetty-launcher-2020-thread-3) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1337806 INFO (jetty-launcher-2020-thread-4) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1337807 INFO (jetty-launcher-2020-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6feccc0a{/solr,null,AVAILABLE}
[junit4] 2> 1337807 INFO (jetty-launcher-2020-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5cffac6c{/solr,null,AVAILABLE}
[junit4] 2> 1337807 INFO (jetty-launcher-2020-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1fd66d09{HTTP/1.1,[http/1.1]}{127.0.0.1:43769}
[junit4] 2> 1337807 INFO (jetty-launcher-2020-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6f06ad{HTTP/1.1,[http/1.1]}{127.0.0.1:56682}
[junit4] 2> 1337808 INFO (jetty-launcher-2020-thread-1) [ ] o.e.j.s.Server Started @1340793ms
[junit4] 2> 1337808 INFO (jetty-launcher-2020-thread-2) [ ] o.e.j.s.Server Started @1340793ms
[junit4] 2> 1337808 INFO (jetty-launcher-2020-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43769}
[junit4] 2> 1337808 INFO (jetty-launcher-2020-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=56682}
[junit4] 2> 1337808 ERROR (jetty-launcher-2020-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1337808 ERROR (jetty-launcher-2020-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1337808 INFO (jetty-launcher-2020-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 1337808 INFO (jetty-launcher-2020-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 1337808 INFO (jetty-launcher-2020-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1337808 INFO (jetty-launcher-2020-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1337808 INFO (jetty-launcher-2020-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1337808 INFO (jetty-launcher-2020-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1337808 INFO (jetty-launcher-2020-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-20T03:41:38.235Z
[junit4] 2> 1337808 INFO (jetty-launcher-2020-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-20T03:41:38.235Z
[junit4] 2> 1337815 INFO (jetty-launcher-2020-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@392a2032{/solr,null,AVAILABLE}
[junit4] 2> 1337815 INFO (jetty-launcher-2020-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@51034b4d{HTTP/1.1,[http/1.1]}{127.0.0.1:49820}
[junit4] 2> 1337815 INFO (jetty-launcher-2020-thread-3) [ ] o.e.j.s.Server Started @1340801ms
[junit4] 2> 1337815 INFO (jetty-launcher-2020-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=49820}
[junit4] 2> 1337815 ERROR (jetty-launcher-2020-thread-3) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1337815 INFO (jetty-launcher-2020-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 1337815 INFO (jetty-launcher-2020-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1337815 INFO (jetty-launcher-2020-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1337816 INFO (jetty-launcher-2020-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-20T03:41:38.243Z
[junit4] 2> 1337828 INFO (jetty-launcher-2020-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1337830 INFO (jetty-launcher-2020-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@e9f5245{/solr,null,AVAILABLE}
[junit4] 2> 1337831 INFO (jetty-launcher-2020-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1bb26d01{HTTP/1.1,[http/1.1]}{127.0.0.1:50563}
[junit4] 2> 1337831 INFO (jetty-launcher-2020-thread-4) [ ] o.e.j.s.Server Started @1340817ms
[junit4] 2> 1337831 INFO (jetty-launcher-2020-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50563}
[junit4] 2> 1337831 ERROR (jetty-launcher-2020-thread-4) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1337835 INFO (jetty-launcher-2020-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 1337835 INFO (jetty-launcher-2020-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1337835 INFO (jetty-launcher-2020-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1337835 INFO (jetty-launcher-2020-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-20T03:41:38.262Z
[junit4] 2> 1337836 INFO (jetty-launcher-2020-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1337838 INFO (jetty-launcher-2020-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1337842 INFO (jetty-launcher-2020-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:49006/solr
[junit4] 2> 1337842 INFO (jetty-launcher-2020-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1337848 INFO (jetty-launcher-2020-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:49006/solr
[junit4] 2> 1337849 INFO (jetty-launcher-2020-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:49006/solr
[junit4] 2> 1337863 INFO (jetty-launcher-2020-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:49006/solr
[junit4] 2> 1337886 INFO (jetty-launcher-2020-thread-3) [n:127.0.0.1:49820_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1337886 INFO (jetty-launcher-2020-thread-3) [n:127.0.0.1:49820_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:49820_solr
[junit4] 2> 1337887 INFO (jetty-launcher-2020-thread-3) [n:127.0.0.1:49820_solr ] o.a.s.c.Overseer Overseer (id=97647414353854471-127.0.0.1:49820_solr-n_0000000000) starting
[junit4] 2> 1337899 INFO (jetty-launcher-2020-thread-1) [n:127.0.0.1:43769_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1337899 INFO (jetty-launcher-2020-thread-2) [n:127.0.0.1:56682_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1337900 INFO (jetty-launcher-2020-thread-4) [n:127.0.0.1:50563_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1337900 INFO (jetty-launcher-2020-thread-1) [n:127.0.0.1:43769_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43769_solr
[junit4] 2> 1337900 INFO (jetty-launcher-2020-thread-2) [n:127.0.0.1:56682_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56682_solr
[junit4] 2> 1337923 INFO (jetty-launcher-2020-thread-4) [n:127.0.0.1:50563_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50563_solr
[junit4] 2> 1337924 INFO (jetty-launcher-2020-thread-3) [n:127.0.0.1:49820_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49820_solr
[junit4] 2> 1337937 INFO (zkCallback-2042-thread-1-processing-n:127.0.0.1:50563_solr) [n:127.0.0.1:50563_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 1337942 INFO (OverseerStateUpdate-97647414353854471-127.0.0.1:49820_solr-n_0000000000) [n:127.0.0.1:49820_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 1337946 INFO (zkCallback-2040-thread-1-processing-n:127.0.0.1:56682_solr) [n:127.0.0.1:56682_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 1337946 INFO (zkCallback-2041-thread-1-processing-n:127.0.0.1:43769_solr) [n:127.0.0.1:43769_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 1338009 INFO (jetty-launcher-2020-thread-4) [n:127.0.0.1:50563_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.OverseerRolesTest_EBA0DE45B66DBB6-001/tempDir-001/node4/.
[junit4] 2> 1338014 INFO (jetty-launcher-2020-thread-2) [n:127.0.0.1:56682_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.OverseerRolesTest_EBA0DE45B66DBB6-001/tempDir-001/node2/.
[junit4] 2> 1338074 INFO (jetty-launcher-2020-thread-3) [n:127.0.0.1:49820_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.OverseerRolesTest_EBA0DE45B66DBB6-001/tempDir-001/node3/.
[junit4] 2> 1338099 INFO (jetty-launcher-2020-thread-1) [n:127.0.0.1:43769_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.OverseerRolesTest_EBA0DE45B66DBB6-001/tempDir-001/node1/.
[junit4] 2> 1339651 INFO (SUITE-OverseerRolesTest-seed#[EBA0DE45B66DBB6]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 1339652 INFO (SUITE-OverseerRolesTest-seed#[EBA0DE45B66DBB6]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:49006/solr ready
[junit4] 2> 1339681 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testOverseerRole
[junit4] 2> 1339681 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.OverseerRolesTest Overseer: 127.0.0.1:49820_solr
[junit4] 2> 1339682 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.OverseerRolesTest Election queue:
[junit4] 2> 1339682 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.OverseerRolesTest ### Setting overseer designate 127.0.0.1:56682_solr
[junit4] 2> 1339686 INFO (qtp597241835-15045) [n:127.0.0.1:50563_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addrole with params node=127.0.0.1:56682_solr&role=overseer&action=ADDROLE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1339688 INFO (OverseerThreadFactory-5864-thread-1-processing-n:127.0.0.1:49820_solr) [n:127.0.0.1:49820_solr ] o.a.s.c.OverseerRoleCmd Overseer role added to 127.0.0.1:56682_solr
[junit4] 2> 1339691 INFO (Thread-5138) [n:127.0.0.1:49820_solr ] o.a.s.c.OverseerNodePrioritizer prioritizing overseer nodes at 97647414353854471-127.0.0.1:49820_solr-n_0000000000 overseer designates are [127.0.0.1:56682_solr]
[junit4] 2> 1339691 INFO (Thread-5138) [n:127.0.0.1:49820_solr ] o.a.s.c.OverseerNodePrioritizer sorted nodes [97647414353854471-127.0.0.1:49820_solr-n_0000000000, 97647414353854474-127.0.0.1:43769_solr-n_0000000001, 97647414353854475-127.0.0.1:56682_solr-n_0000000002, 97647414353854476-127.0.0.1:50563_solr-n_0000000003]
[junit4] 2> 1339691 INFO (Thread-5138) [n:127.0.0.1:49820_solr ] o.a.s.c.OverseerNodePrioritizer asking node 97647414353854475-127.0.0.1:56682_solr-n_0000000002 to come join election at head
[junit4] 2> 1339691 INFO (qtp597241835-15045) [n:127.0.0.1:50563_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:56682_solr&role=overseer&action=ADDROLE&wt=javabin&version=2} status=0 QTime=5
[junit4] 2> 1339693 INFO (qtp1496172340-15032) [n:127.0.0.1:56682_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={op=rejoinAtHead&qt=/admin/cores&action=OVERSEEROP&electionNode=97647414353854475-127.0.0.1:56682_solr-n_0000000002&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1339693 INFO (Thread-5138) [n:127.0.0.1:49820_solr ] o.a.s.c.OverseerNodePrioritizer asking the old first in line 97647414353854474-127.0.0.1:43769_solr-n_0000000001 to rejoin election
[junit4] 2> 1339694 INFO (qtp950248194-15042) [n:127.0.0.1:43769_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={op=rejoin&qt=/admin/cores&action=OVERSEEROP&electionNode=97647414353854474-127.0.0.1:43769_solr-n_0000000001&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1339696 INFO (OverseerStateUpdate-97647414353854471-127.0.0.1:49820_solr-n_0000000000) [n:127.0.0.1:49820_solr ] o.a.s.c.Overseer Quit command received {
[junit4] 2> "operation":"quit",
[junit4] 2> "id":"97647414353854471-127.0.0.1:49820_solr-n_0000000000"} 127.0.0.1:49820_solr
[junit4] 2> 1339697 INFO (OverseerStateUpdate-97647414353854471-127.0.0.1:49820_solr-n_0000000000) [n:127.0.0.1:49820_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:49820_solr
[junit4] 2> 1339697 WARN (OverseerExitThread) [n:127.0.0.1:49820_solr ] o.a.s.c.Overseer I'm exiting, but I'm still the leader
[junit4] 2> 1339698 INFO (OverseerExitThread) [n:127.0.0.1:49820_solr ] o.a.s.c.Overseer Overseer (id=97647414353854471-127.0.0.1:49820_solr-n_0000000000) closing
[junit4] 2> 1339698 INFO (zkCallback-2041-thread-1-processing-n:127.0.0.1:43769_solr) [n:127.0.0.1:43769_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43769_solr
[junit4] 2> 1339699 INFO (zkCallback-2041-thread-1-processing-n:127.0.0.1:43769_solr) [n:127.0.0.1:43769_solr ] o.a.s.c.Overseer Overseer (id=97647414353854474-127.0.0.1:43769_solr-n_0000000001) starting
[junit4] 2> 1339704 INFO (OverseerCollectionConfigSetProcessor-97647414353854474-127.0.0.1:43769_solr-n_0000000001) [n:127.0.0.1:43769_solr ] o.a.s.c.OverseerNodePrioritizer prioritizing overseer nodes at 97647414353854474-127.0.0.1:43769_solr-n_0000000001 overseer designates are [127.0.0.1:56682_solr]
[junit4] 2> 1339715 INFO (OverseerCollectionConfigSetProcessor-97647414353854474-127.0.0.1:43769_solr-n_0000000001) [n:127.0.0.1:43769_solr ] o.a.s.c.OverseerNodePrioritizer sorted nodes [97647414353854474-127.0.0.1:43769_solr-n_0000000001, 97647414353854475-127.0.0.1:56682_solr-n_0000000002, 97647414353854476-127.0.0.1:50563_solr-n_0000000003, 97647414353854471-127.0.0.1:49820_solr-n_0000000004]
[junit4] 2> 1339718 INFO (OverseerStateUpdate-97647414353854474-127.0.0.1:43769_solr-n_0000000001) [n:127.0.0.1:43769_solr ] o.a.s.c.Overseer Quit command received {
[junit4] 2> "operation":"quit",
[junit4] 2> "id":"97647414353854474-127.0.0.1:43769_solr-n_0000000001"} 127.0.0.1:43769_solr
[junit4] 2> 1339724 INFO (OverseerStateUpdate-97647414353854474-127.0.0.1:43769_solr-n_0000000001) [n:127.0.0.1:43769_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:43769_solr
[junit4] 2> 1339726 WARN (OverseerExitThread) [n:127.0.0.1:43769_solr ] o.a.s.c.Overseer I'm exiting, but I'm still the leader
[junit4] 2> 1339727 INFO (OverseerExitThread) [n:127.0.0.1:43769_solr ] o.a.s.c.Overseer Overseer (id=97647414353854474-127.0.0.1:43769_solr-n_0000000001) closing
[junit4] 2> 1339727 INFO (zkCallback-2040-thread-1-processing-n:127.0.0.1:56682_solr) [n:127.0.0.1:56682_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56682_solr
[junit4] 2> 1339729 INFO (zkCallback-2040-thread-1-processing-n:127.0.0.1:56682_solr) [n:127.0.0.1:56682_solr ] o.a.s.c.Overseer Overseer (id=97647414353854475-127.0.0.1:56682_solr-n_0000000002) starting
[junit4] 2> 1339794 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.OverseerRolesTest ### Adding another overseer designate 127.0.0.1:50563_solr
[junit4] 2> 1339794 INFO (qtp597241835-15064) [n:127.0.0.1:50563_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addrole with params node=127.0.0.1:50563_solr&role=overseer&action=ADDROLE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1339803 INFO (OverseerThreadFactory-5876-thread-1-processing-n:127.0.0.1:56682_solr) [n:127.0.0.1:56682_solr ] o.a.s.c.OverseerRoleCmd Overseer role added to 127.0.0.1:50563_solr
[junit4] 2> 1339819 INFO (qtp597241835-15064) [n:127.0.0.1:50563_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:50563_solr&role=overseer&action=ADDROLE&wt=javabin&version=2} status=0 QTime=24
[junit4] 2> 1339823 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.OverseerRolesTest Overseer: 127.0.0.1:56682_solr
[junit4] 2> 1339823 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.OverseerRolesTest Election queue:
[junit4] 2> 1339823 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 56682
[junit4] 2> 1339823 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=2080453031
[junit4] 2> 1339823 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 1339826 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 1339828 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.Overseer Overseer (id=97647414353854475-127.0.0.1:56682_solr-n_0000000002) closing
[junit4] 2> 1339832 INFO (OverseerStateUpdate-97647414353854475-127.0.0.1:56682_solr-n_0000000002) [n:127.0.0.1:56682_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:56682_solr
[junit4] 2> 1339852 INFO (zkCallback-2036-thread-2-processing-n:127.0.0.1:49820_solr) [n:127.0.0.1:49820_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 1339852 INFO (zkCallback-2041-thread-1-processing-n:127.0.0.1:43769_solr) [n:127.0.0.1:43769_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 1339852 INFO (zkCallback-2042-thread-1-processing-n:127.0.0.1:50563_solr) [n:127.0.0.1:50563_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:50563_solr
[junit4] 2> 1339853 INFO (zkCallback-2042-thread-1-processing-n:127.0.0.1:50563_solr) [n:127.0.0.1:50563_solr ] o.a.s.c.Overseer Overseer (id=97647414353854476-127.0.0.1:50563_solr-n_0000000003) starting
[junit4] 2> 1339861 INFO (zkCallback-2050-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 1339861 INFO (zkCallback-2042-thread-2-processing-n:127.0.0.1:50563_solr) [n:127.0.0.1:50563_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 1341351 WARN (zkCallback-2040-thread-1-processing-n:127.0.0.1:56682_solr) [n:127.0.0.1:56682_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1341352 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@6f06ad{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 1341353 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6feccc0a{/solr,null,UNAVAILABLE}
[junit4] 2> 1341354 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.OverseerRolesTest ### Adding last overseer designate 127.0.0.1:43769_solr
[junit4] 2> 1341355 INFO (qtp597241835-15067) [n:127.0.0.1:50563_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addrole with params node=127.0.0.1:43769_solr&role=overseer&action=ADDROLE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1341360 INFO (OverseerThreadFactory-5879-thread-1-processing-n:127.0.0.1:50563_solr) [n:127.0.0.1:50563_solr ] o.a.s.c.OverseerRoleCmd Overseer role added to 127.0.0.1:43769_solr
[junit4] 2> 1341362 INFO (qtp597241835-15067) [n:127.0.0.1:50563_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:43769_solr&role=overseer&action=ADDROLE&wt=javabin&version=2} status=0 QTime=7
[junit4] 2> 1341366 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.OverseerRolesTest Overseer: 127.0.0.1:50563_solr
[junit4] 2> 1341367 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.OverseerRolesTest Election queue:
[junit4] 2> 1341369 INFO (qtp597241835-15068) [n:127.0.0.1:50563_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :removerole with params node=127.0.0.1:50563_solr&role=overseer&action=REMOVEROLE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1341370 INFO (OverseerCollectionConfigSetProcessor-97647414353854476-127.0.0.1:50563_solr-n_0000000003) [n:127.0.0.1:50563_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1341371 INFO (OverseerThreadFactory-5879-thread-2-processing-n:127.0.0.1:50563_solr) [n:127.0.0.1:50563_solr ] o.a.s.c.OverseerRoleCmd Overseer role removed from 127.0.0.1:50563_solr
[junit4] 2> 1341375 INFO (qtp597241835-15068) [n:127.0.0.1:50563_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:50563_solr&role=overseer&action=REMOVEROLE&wt=javabin&version=2} status=0 QTime=7
[junit4] 2> 1341376 INFO (Thread-5142) [n:127.0.0.1:50563_solr ] o.a.s.c.OverseerNodePrioritizer prioritizing overseer nodes at 97647414353854476-127.0.0.1:50563_solr-n_0000000003 overseer designates are [127.0.0.1:56682_solr, 127.0.0.1:43769_solr]
[junit4] 2> 1341376 INFO (Thread-5142) [n:127.0.0.1:50563_solr ] o.a.s.c.OverseerNodePrioritizer sorted nodes [97647414353854476-127.0.0.1:50563_solr-n_0000000003, 97647414353854471-127.0.0.1:49820_solr-n_0000000004, 97647414353854474-127.0.0.1:43769_solr-n_0000000005]
[junit4] 2> 1341376 INFO (Thread-5142) [n:127.0.0.1:50563_solr ] o.a.s.c.OverseerNodePrioritizer asking node 97647414353854474-127.0.0.1:43769_solr-n_0000000005 to come join election at head
[junit4] 2> 1341377 INFO (qtp950248194-15047) [n:127.0.0.1:43769_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={op=rejoinAtHead&qt=/admin/cores&action=OVERSEEROP&electionNode=97647414353854474-127.0.0.1:43769_solr-n_0000000005&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1341377 INFO (Thread-5142) [n:127.0.0.1:50563_solr ] o.a.s.c.OverseerNodePrioritizer asking the old first in line 97647414353854471-127.0.0.1:49820_solr-n_0000000004 to rejoin election
[junit4] 2> 1341378 INFO (qtp842075566-15052) [n:127.0.0.1:49820_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={op=rejoin&qt=/admin/cores&action=OVERSEEROP&electionNode=97647414353854471-127.0.0.1:49820_solr-n_0000000004&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1341379 INFO (OverseerStateUpdate-97647414353854476-127.0.0.1:50563_solr-n_0000000003) [n:127.0.0.1:50563_solr ] o.a.s.c.Overseer Quit command received {
[junit4] 2> "operation":"quit",
[junit4] 2> "id":"97647414353854476-127.0.0.1:50563_solr-n_0000000003"} 127.0.0.1:50563_solr
[junit4] 2> 1341380 INFO (OverseerStateUpdate-97647414353854476-127.0.0.1:50563_solr-n_0000000003) [n:127.0.0.1:50563_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:50563_solr
[junit4] 2> 1341383 WARN (OverseerExitThread) [n:127.0.0.1:50563_solr ] o.a.s.c.Overseer I'm exiting, but I'm still the leader
[junit4] 2> 1341383 INFO (OverseerExitThread) [n:127.0.0.1:50563_solr ] o.a.s.c.Overseer Overseer (id=97647414353854476-127.0.0.1:50563_solr-n_0000000003) closing
[junit4] 2> 1341385 INFO (zkCallback-2036-thread-2-processing-n:127.0.0.1:49820_solr) [n:127.0.0.1:49820_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:49820_solr
[junit4] 2> 1341386 INFO (zkCallback-2036-thread-2-processing-n:127.0.0.1:49820_solr) [n:127.0.0.1:49820_solr ] o.a.s.c.Overseer Overseer (id=97647414353854471-127.0.0.1:49820_solr-n_0000000004) starting
[junit4] 2> 1341392 INFO (OverseerCollectionConfigSetProcessor-97647414353854471-127.0.0.1:49820_solr-n_0000000004) [n:127.0.0.1:49820_solr ] o.a.s.c.OverseerNodePrioritizer prioritizing overseer nodes at 97647414353854471-127.0.0.1:49820_solr-n_0000000004 overseer designates are [127.0.0.1:56682_solr, 127.0.0.1:43769_solr]
[junit4] 2> 1341392 INFO (OverseerCollectionConfigSetProcessor-97647414353854471-127.0.0.1:49820_solr-n_0000000004) [n:127.0.0.1:49820_solr ] o.a.s.c.OverseerNodePrioritizer sorted nodes [97647414353854471-127.0.0.1:49820_solr-n_0000000004, 97647414353854474-127.0.0.1:43769_solr-n_0000000005, 97647414353854476-127.0.0.1:50563_solr-n_0000000006]
[junit4] 2> 1341394 INFO (OverseerStateUpdate-97647414353854471-127.0.0.1:49820_solr-n_0000000004) [n:127.0.0.1:49820_solr ] o.a.s.c.Overseer Quit command received {
[junit4] 2> "operation":"quit",
[junit4] 2> "id":"97647414353854471-127.0.0.1:49820_solr-n_0000000004"} 127.0.0.1:49820_solr
[junit4] 2> 1341395 INFO (OverseerStateUpdate-97647414353854471-127.0.0.1:49820_solr-n_0000000004) [n:127.0.0.1:49820_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:49820_solr
[junit4] 2> 1341395 WARN (OverseerExitThread) [n:127.0.0.1:49820_solr ] o.a.s.c.Overseer I'm exiting, but I'm still the leader
[junit4] 2> 1341396 INFO (OverseerExitThread) [n:127.0.0.1:49820_solr ] o.a.s.c.Overseer Overseer (id=97647414353854471-127.0.0.1:49820_solr-n_0000000004) closing
[junit4] 2> 1341396 INFO (zkCallback-2041-thread-1-processing-n:127.0.0.1:43769_solr) [n:127.0.0.1:43769_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43769_solr
[junit4] 2> 1341398 INFO (zkCallback-2041-thread-1-processing-n:127.0.0.1:43769_solr) [n:127.0.0.1:43769_solr ] o.a.s.c.Overseer Overseer (id=97647414353854474-127.0.0.1:43769_solr-n_0000000005) starting
[junit4] 2> 1341477 INFO (qtp597241835-15069) [n:127.0.0.1:50563_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addrole with params node=127.0.0.1:50563_solr&role=overseer&action=ADDROLE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1341482 INFO (OverseerThreadFactory-5883-thread-1-processing-n:127.0.0.1:43769_solr) [n:127.0.0.1:43769_solr ] o.a.s.c.OverseerRoleCmd Overseer role added to 127.0.0.1:50563_solr
[junit4] 2> 1341491 INFO (qtp597241835-15069) [n:127.0.0.1:50563_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:50563_solr&role=overseer&action=ADDROLE&wt=javabin&version=2} status=0 QTime=14
[junit4] 2> 1341491 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.OverseerRolesTest ### Sending QUIT to overseer 127.0.0.1:43769_solr
[junit4] 2> 1341492 INFO (OverseerStateUpdate-97647414353854474-127.0.0.1:43769_solr-n_0000000005) [n:127.0.0.1:43769_solr ] o.a.s.c.Overseer Quit command received {
[junit4] 2> "operation":"quit",
[junit4] 2> "id":"97647414353854474-127.0.0.1:43769_solr-n_0000000005"} 127.0.0.1:43769_solr
[junit4] 2> 1341493 INFO (OverseerStateUpdate-97647414353854474-127.0.0.1:43769_solr-n_0000000005) [n:127.0.0.1:43769_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:43769_solr
[junit4] 2> 1341506 WARN (OverseerExitThread) [n:127.0.0.1:43769_solr ] o.a.s.c.Overseer I'm exiting, but I'm still the leader
[junit4] 2> 1341507 INFO (Thread-5143) [n:127.0.0.1:43769_solr ] o.a.s.c.OverseerNodePrioritizer prioritizing overseer nodes at 97647414353854474-127.0.0.1:43769_solr-n_0000000005 overseer designates are [127.0.0.1:56682_solr, 127.0.0.1:43769_solr, 127.0.0.1:50563_solr]
[junit4] 2> 1341507 INFO (Thread-5143) [n:127.0.0.1:43769_solr ] o.a.s.c.OverseerNodePrioritizer sorted nodes [97647414353854474-127.0.0.1:43769_solr-n_0000000005, 97647414353854476-127.0.0.1:50563_solr-n_0000000006, 97647414353854471-127.0.0.1:49820_solr-n_0000000007]
[junit4] 2> 1341507 INFO (Thread-5143) [n:127.0.0.1:43769_solr ] o.a.s.c.OverseerNodePrioritizer asking node 97647414353854474-127.0.0.1:43769_solr-n_0000000005 to come join election at head
[junit4] 2> 1341508 INFO (OverseerExitThread) [n:127.0.0.1:43769_solr ] o.a.s.c.Overseer Overseer (id=97647414353854474-127.0.0.1:43769_solr-n_0000000005) closing
[junit4] 2> 1341511 INFO (zkCallback-2042-thread-1-processing-n:127.0.0.1:50563_solr) [n:127.0.0.1:50563_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:50563_solr
[junit4] 2> 1341511 INFO (qtp950248194-15048) [n:127.0.0.1:43769_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={op=rejoinAtHead&qt=/admin/cores&action=OVERSEEROP&electionNode=97647414353854474-127.0.0.1:43769_solr-n_0000000005&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1341512 INFO (zkCallback-2042-thread-1-processing-n:127.0.0.1:50563_solr) [n:127.0.0.1:50563_solr ] o.a.s.c.Overseer Overseer (id=97647414353854476-127.0.0.1:50563_solr-n_0000000006) starting
[junit4] 2> 1341512 INFO (Thread-5143) [n:127.0.0.1:43769_solr ] o.a.s.c.OverseerNodePrioritizer asking the old first in line 97647414353854476-127.0.0.1:50563_solr-n_0000000006 to rejoin election
[junit4] 2> 1341520 INFO (qtp597241835-15045) [n:127.0.0.1:50563_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={op=rejoin&qt=/admin/cores&action=OVERSEEROP&electionNode=97647414353854476-127.0.0.1:50563_solr-n_0000000006&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1341524 INFO (OverseerStateUpdate-97647414353854476-127.0.0.1:50563_solr-n_0000000006) [n:127.0.0.1:50563_solr ] o.a.s.c.Overseer Quit command received {
[junit4] 2> "operation":"quit",
[junit4] 2> "id":"97647414353854476-127.0.0.1:50563_solr-n_0000000006"} 127.0.0.1:50563_solr
[junit4] 2> 1341525 INFO (OverseerStateUpdate-97647414353854476-127.0.0.1:50563_solr-n_0000000006) [n:127.0.0.1:50563_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:50563_solr
[junit4] 2> 1341533 WARN (OverseerExitThread) [n:127.0.0.1:50563_solr ] o.a.s.c.Overseer I'm exiting, but I'm still the leader
[junit4] 2> 1341533 INFO (OverseerExitThread) [n:127.0.0.1:50563_solr ] o.a.s.c.Overseer Overseer (id=97647414353854476-127.0.0.1:50563_solr-n_0000000006) closing
[junit4] 2> 1341535 INFO (zkCallback-2036-thread-2-processing-n:127.0.0.1:49820_solr) [n:127.0.0.1:49820_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:49820_solr
[junit4] 2> 1341536 INFO (zkCallback-2036-thread-2-processing-n:127.0.0.1:49820_solr) [n:127.0.0.1:49820_solr ] o.a.s.c.Overseer Overseer (id=97647414353854471-127.0.0.1:49820_solr-n_0000000007) starting
[junit4] 2> 1341543 INFO (OverseerCollectionConfigSetProcessor-97647414353854471-127.0.0.1:49820_solr-n_0000000007) [n:127.0.0.1:49820_solr ] o.a.s.c.OverseerNodePrioritizer prioritizing overseer nodes at 97647414353854471-127.0.0.1:49820_solr-n_0000000007 overseer designates are [127.0.0.1:56682_solr, 127.0.0.1:43769_solr, 127.0.0.1:50563_solr]
[junit4] 2> 1341543 INFO (OverseerCollectionConfigSetProcessor-97647414353854471-127.0.0.1:49820_solr-n_0000000007) [n:127.0.0.1:49820_solr ] o.a.s.c.OverseerNodePrioritizer sorted nodes [97647414353854471-127.0.0.1:49820_solr-n_0000000007, 97647414353854474-127.0.0.1:43769_solr-n_0000000008, 97647414353854476-127.0.0.1:50563_solr-n_0000000009]
[junit4] 2> 1341544 INFO (OverseerStateUpdate-97647414353854471-127.0.0.1:49820_solr-n_0000000007) [n:127.0.0.1:49820_solr ] o.a.s.c.Overseer Quit command received {
[junit4] 2> "operation":"quit",
[junit4] 2> "id":"97647414353854471-127.0.0.1:49820_solr-n_0000000007"} 127.0.0.1:49820_solr
[junit4] 2> 1341545 INFO (OverseerStateUpdate-97647414353854471-127.0.0.1:49820_solr-n_0000000007) [n:127.0.0.1:49820_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:49820_solr
[junit4] 2> 1341557 WARN (OverseerExitThread) [n:127.0.0.1:49820_solr ] o.a.s.c.Overseer I'm exiting, but I'm still the leader
[junit4] 2> 1341558 INFO (OverseerExitThread) [n:127.0.0.1:49820_solr ] o.a.s.c.Overseer Overseer (id=97647414353854471-127.0.0.1:49820_solr-n_0000000007) closing
[junit4] 2> 1341558 INFO (zkCallback-2041-thread-1-processing-n:127.0.0.1:43769_solr) [n:127.0.0.1:43769_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43769_solr
[junit4] 2> 1341565 INFO (zkCallback-2041-thread-1-processing-n:127.0.0.1:43769_solr) [n:127.0.0.1:43769_solr ] o.a.s.c.Overseer Overseer (id=97647414353854474-127.0.0.1:43769_solr-n_0000000008) starting
[junit4] 2> 1356567 INFO (TEST-OverseerRolesTest.testOverseerRole-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testOverseerRole
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=OverseerRolesTest -Dtests.method=testOverseerRole -Dtests.seed=EBA0DE45B66DBB6 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=th -Dtests.timezone=Africa/Kampala -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 16.9s J2 | OverseerRolesTest.testOverseerRole <<<
[junit4] > Throwable #1: java.lang.AssertionError: Timed out waiting for overseer state change
[junit4] > at __randomizedtesting.SeedInfo.seed([EBA0DE45B66DBB6:EF71F07060D5ED67]:0)
[junit4] > at org.apache.solr.cloud.OverseerRolesTest.waitForNewOverseer(OverseerRolesTest.java:62)
[junit4] > at org.apache.solr.cloud.OverseerRolesTest.testOverseerRole(OverseerRolesTest.java:140)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 1356572 INFO (jetty-closer-2021-thread-2) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@1bb26d01{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 1356573 INFO (jetty-closer-2021-thread-3) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@51034b4d{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 1356573 INFO (jetty-closer-2021-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@1fd66d09{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 1356573 INFO (jetty-closer-2021-thread-2) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=811645320
[junit4] 2> 1356574 INFO (jetty-closer-2021-thread-3) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1236941701
[junit4] 2> 1356574 INFO (jetty-closer-2021-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 1356574 INFO (jetty-closer-2021-thread-3) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 1356574 INFO (jetty-closer-2021-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=269790375
[junit4] 2> 1356574 INFO (jetty-closer-2021-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 1356575 INFO (jetty-closer-2021-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 1356576 INFO (zkCallback-2036-thread-2-processing-n:127.0.0.1:49820_solr) [n:127.0.0.1:49820_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 1356576 INFO (zkCallback-2041-thread-1-processing-n:127.0.0.1:43769_solr) [n:127.0.0.1:43769_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 1356577 INFO (jetty-closer-2021-thread-3) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 1356578 INFO (zkCallback-2041-thread-1-processing-n:127.0.0.1:43769_solr) [n:127.0.0.1:43769_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 1356583 INFO (jetty-closer-2021-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 1356583 INFO (jetty-closer-2021-thread-1) [ ] o.a.s.c.Overseer Overseer (id=97647414353854474-127.0.0.1:43769_solr-n_0000000008) closing
[junit4] 2> 1356583 INFO (OverseerStateUpdate-97647414353854474-127.0.0.1:43769_solr-n_0000000008) [n:127.0.0.1:43769_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:43769_solr
[junit4] 2> 1356585 WARN (zkCallback-2041-thread-2-processing-n:127.0.0.1:43769_solr) [n:127.0.0.1:43769_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1356585 INFO (jetty-closer-2021-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5cffac6c{/solr,null,UNAVAILABLE}
[junit4] 2> 1358076 WARN (zkCallback-2042-thread-1-processing-n:127.0.0.1:50563_solr) [n:127.0.0.1:50563_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1358077 INFO (jetty-closer-2021-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@e9f5245{/solr,null,UNAVAILABLE}
[junit4] 2> 1358078 WARN (zkCallback-2036-thread-2-processing-n:127.0.0.1:49820_solr) [n:127.0.0.1:49820_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1358078 INFO (jetty-closer-2021-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@392a2032{/solr,null,UNAVAILABLE}
[junit4] 2> 1358083 INFO (SUITE-OverseerRolesTest-seed#[EBA0DE45B66DBB6]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:49006 49006
[junit4] 2> 1358268 INFO (Thread-5101) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:49006 49006
[junit4] 2> 1358268 WARN (Thread-5101) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/aliases.json
[junit4] 2> 5 /solr/clusterprops.json
[junit4] 2> 4 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/clusterstate.json
[junit4] 2> 2 /solr/overseer_elect/election/97647414353854474-127.0.0.1:43769_solr-n_0000000008
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 19 /solr/overseer/collection-queue-work
[junit4] 2> 17 /solr/overseer/queue-work
[junit4] 2> 5 /solr/live_nodes
[junit4] 2> 5 /solr/collections
[junit4] 2> 4 /solr/overseer/queue
[junit4] 2>
[junit4] 2> 1358269 INFO (SUITE-OverseerRolesTest-seed#[EBA0DE45B66DBB6]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.OverseerRolesTest_EBA0DE45B66DBB6-001
[junit4] 2> Mar 20, 2017 3:41:58 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=10969, maxDocsPerChunk=273, blockSize=308), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=10969, blockSize=308)), sim=RandomSimilarity(queryNorm=false): {}, locale=th, timezone=Africa/Kampala
[junit4] 2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=4,threads=1,free=152549464,total=522715136
[junit4] 2> NOTE: All tests run in this JVM: [TestReRankQParserPlugin, DeleteLastCustomShardedReplicaTest, TestConfigsApi, SuggestComponentContextFilterQueryTest, TestGroupingSearch, BlockCacheTest, TestFieldCacheSortRandom, SimpleFacetsTest, SolrCLIZkUtilsTest, BinaryUpdateRequestHandlerTest, TestIndexSearcher, TestZkChroot, TestUniqueKeyFieldResource, TestConfig, TestSolrDeletionPolicy1, HLLUtilTest, DistributedFacetPivotLongTailTest, TestSolr4Spatial2, PrimitiveFieldTypeTest, TestRestManager, DistributedSuggestComponentTest, TestDistributedMissingSort, TestIndexingPerformance, TestSegmentSorting, TestConfigSets, HighlighterConfigTest, RecoveryZkTest, PreAnalyzedFieldTest, JSONWriterTest, TestManagedSchemaAPI, OutputWriterTest, TestUseDocValuesAsStored2, HdfsRecoveryZkTest, StatsComponentTest, AutoCommitTest, TriLevelCompositeIdRoutingTest, ShowFileRequestHandlerTest, CdcrVersionReplicationTest, TestStressCloudBlindAtomicUpdates, BadIndexSchemaTest, TestComponentsName, TestStandardQParsers, TestMiniSolrCloudClusterSSL, ConcurrentDeleteAndCreateCollectionTest, TestCloudSchemaless, TestLeaderElectionZkExpiry, CdcrReplicationDistributedZkTest, BadCopyFieldTest, TestSolrXml, LeaderElectionTest, TestChildDocTransformer, RuleEngineTest, TestCloudPivotFacet, TestDeleteCollectionOnDownNodes, TestDocSet, TestTolerantUpdateProcessorCloud, TestCSVResponseWriter, TestFastOutputStream, MBeansHandlerTest, CheckHdfsIndexTest, V2ApiIntegrationTest, SolrPluginUtilsTest, SolrCoreMetricManagerTest, SuggesterTSTTest, HdfsCollectionsAPIDistributedZkTest, UpdateLogTest, DebugComponentTest, HardAutoCommitTest, CoreMergeIndexesAdminHandlerTest, TermVectorComponentDistributedTest, BlockJoinFacetRandomTest, HdfsThreadLeakTest, SolrTestCaseJ4Test, StatelessScriptUpdateProcessorFactoryTest, ResourceLoaderTest, SliceStateTest, DocValuesMultiTest, EchoParamsTest, TestSolrCloudWithHadoopAuthPlugin, ManagedSchemaRoundRobinCloudTest, DirectUpdateHandlerTest, ReplicationFactorTest, CreateCollectionCleanupTest, MigrateRouteKeyTest, TestCollationField, XmlUpdateRequestHandlerTest, SharedFSAutoReplicaFailoverTest, TestDynamicFieldResource, TestExactSharedStatsCache, StatsReloadRaceTest, TestUninvertingReader, TestClusterProperties, HdfsDirectoryFactoryTest, DistributedFacetPivotWhiteBoxTest, CursorMarkTest, TestSolrQueryParser, TestMultiWordSynonyms, TestSortByMinMaxFunction, TestLockTree, TestRecovery, TestQueryWrapperFilter, BasicDistributedZk2Test, HttpPartitionTest, FullSolrCloudDistribCmdsTest, TestHdfsBackupRestoreCore, TestSolrCloudWithDelegationTokens, TestRequestForwarding, TestCoreDiscovery, SynonymTokenizerTest, TestScoreJoinQPNoScore, BasicAuthIntegrationTest, TestDFISimilarityFactory, LeaderElectionIntegrationTest, QueryElevationComponentTest, TestLazyCores, TestBadConfig, HighlighterTest, TestExtendedDismaxParser, SolrRequestParserTest, TestFoldingMultitermQuery, SuggesterTest, TestStressLucene, TestCSVLoader, SchemaVersionSpecificBehaviorTest, TestUpdate, FieldMutatingUpdateProcessorTest, DirectUpdateHandlerOptimizeTest, SolrInfoMBeanTest, IndexBasedSpellCheckerTest, DisMaxRequestHandlerTest, TestQueryUtils, TestWriterPerf, DocumentAnalysisRequestHandlerTest, TestOmitPositions, TestValueSourceCache, FieldAnalysisRequestHandlerTest, ReturnFieldsTest, TestPHPSerializedResponseWriter, ResponseLogComponentTest, SolrIndexConfigTest, MultiTermTest, SampleTest, TestBinaryField, NumericFieldsTest, DirectSolrConnectionTest, TestLuceneMatchVersion, TestSweetSpotSimilarityFactory, TestLMDirichletSimilarityFactory, TestIBSimilarityFactory, TestJmxMonitoredMap, URLClassifyProcessorTest, PrimUtilsTest, DateFieldTest, ClusterStateTest, SystemInfoHandlerTest, CircularListTest, DistributedIntervalFacetingTest, AliasIntegrationTest, CdcrReplicationHandlerTest, CdcrRequestHandlerTest, CloudExitableDirectoryReaderTest, CollectionStateFormat2Test, CollectionsAPIAsyncDistributedZkTest, DeleteInactiveReplicaTest, DeleteNodeTest, DistribCursorPagingTest, LeaderFailoverAfterPartitionTest, MissingSegmentRecoveryTest, OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, OverseerRolesTest]
[junit4] Completed [502/701 (1!)] on J2 in 21.09s, 1 test, 1 failure <<< FAILURES!
[...truncated 561 lines...]
[junit4] Suite: org.apache.solr.cloud.ShardSplitTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_EBA0DE45B66DBB6-001/init-core-data-001
[junit4] 2> 1384047 WARN (SUITE-ShardSplitTest-seed#[EBA0DE45B66DBB6]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=8 numCloses=8
[junit4] 2> 1384047 INFO (SUITE-ShardSplitTest-seed#[EBA0DE45B66DBB6]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields
[junit4] 2> 1384048 INFO (SUITE-ShardSplitTest-seed#[EBA0DE45B66DBB6]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 1384048 INFO (SUITE-ShardSplitTest-seed#[EBA0DE45B66DBB6]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /_rbk/f
[junit4] 2> 1384050 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1384054 INFO (Thread-5202) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1384054 INFO (Thread-5202) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1384154 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.ZkTestServer start zk server on port:48552
[junit4] 2> 1384164 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1384166 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 1384167 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1384168 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1384169 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1384170 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1384170 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1384171 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1384172 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1384173 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1384174 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1384244 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_EBA0DE45B66DBB6-001/control-001/cores/collection1
[junit4] 2> 1384245 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1384246 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4108d2e8{/_rbk/f,null,AVAILABLE}
[junit4] 2> 1384247 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@56b29631{HTTP/1.1,[http/1.1]}{127.0.0.1:44327}
[junit4] 2> 1384247 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.e.j.s.Server Started @1387233ms
[junit4] 2> 1384247 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_EBA0DE45B66DBB6-001/tempDir-001/control/data, hostContext=/_rbk/f, hostPort=44327, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_EBA0DE45B66DBB6-001/control-001/cores}
[junit4] 2> 1384248 ERROR (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1384248 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 1384248 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1384248 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1384248 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-20T03:42:24.675Z
[junit4] 2> 1384258 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1384258 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_EBA0DE45B66DBB6-001/control-001/solr.xml
[junit4] 2> 1384264 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48552/solr
[junit4] 2> 1384280 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [n:127.0.0.1:44327__rbk%2Ff ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1384281 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [n:127.0.0.1:44327__rbk%2Ff ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:44327__rbk%2Ff
[junit4] 2> 1384281 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [n:127.0.0.1:44327__rbk%2Ff ] o.a.s.c.Overseer Overseer (id=97647417391972356-127.0.0.1:44327__rbk%2Ff-n_0000000000) starting
[junit4] 2> 1384286 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [n:127.0.0.1:44327__rbk%2Ff ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44327__rbk%2Ff
[junit4] 2> 1384288 INFO (OverseerStateUpdate-97647417391972356-127.0.0.1:44327__rbk%2Ff-n_0000000000) [n:127.0.0.1:44327__rbk%2Ff ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1384323 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [n:127.0.0.1:44327__rbk%2Ff ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_EBA0DE45B66DBB6-001/control-001/cores
[junit4] 2> 1384323 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [n:127.0.0.1:44327__rbk%2Ff ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1384328 INFO (OverseerStateUpdate-97647417391972356-127.0.0.1:44327__rbk%2Ff-n_0000000000) [n:127.0.0.1:44327__rbk%2Ff ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1385338 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1385349 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1385415 WARN (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1385416 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1385434 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 1385434 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1385435 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_EBA0DE45B66DBB6-001/control-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_EBA0DE45B66DBB6-001/control-001/cores/collection1/data/]
[junit4] 2> 1385435 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@97ad257
[junit4] 2> 1385436 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=22, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 1385445 WARN (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1385471 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1385471 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1385472 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1385472 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1385472 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=36, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 1385473 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@705688d8[collection1] main]
[junit4] 2> 1385476 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1385477 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1385477 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1385478 INFO (coreLoadExecutor-5973-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1562358679763681280
[junit4] 2> 1385483 INFO (searcherExecutor-5974-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff x:collection1 c:control_collection) [n:127.0.0.1:44327__rbk%2Ff c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@705688d8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1385487 INFO (coreZkRegister-5966-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff x:collection1 c:control_collection) [n:127.0.0.1:44327__rbk%2Ff c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1385487 INFO (coreZkRegister-5966-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff x:collection1 c:control_collection) [n:127.0.0.1:44327__rbk%2Ff c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1385487 INFO (coreZkRegister-5966-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff x:collection1 c:control_collection) [n:127.0.0.1:44327__rbk%2Ff c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:44327/_rbk/f/collection1/
[junit4] 2> 1385487 INFO (coreZkRegister-5966-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff x:collection1 c:control_collection) [n:127.0.0.1:44327__rbk%2Ff c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1385487 INFO (coreZkRegister-5966-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff x:collection1 c:control_collection) [n:127.0.0.1:44327__rbk%2Ff c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:44327/_rbk/f/collection1/ has no replicas
[junit4] 2> 1385487 INFO (coreZkRegister-5966-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff x:collection1 c:control_collection) [n:127.0.0.1:44327__rbk%2Ff c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1385490 INFO (coreZkRegister-5966-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff x:collection1 c:control_collection) [n:127.0.0.1:44327__rbk%2Ff c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:44327/_rbk/f/collection1/ shard1
[junit4] 2> 1385641 INFO (coreZkRegister-5966-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff x:collection1 c:control_collection) [n:127.0.0.1:44327__rbk%2Ff c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1385868 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1385868 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48552/solr ready
[junit4] 2> 1385868 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1385941 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_EBA0DE45B66DBB6-001/shard-1-001/cores/collection1
[junit4] 2> 1385942 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_EBA0DE45B66DBB6-001/shard-1-001
[junit4] 2> 1385942 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1385943 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4a7f0f96{/_rbk/f,null,AVAILABLE}
[junit4] 2> 1385943 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@623d2a32{HTTP/1.1,[http/1.1]}{127.0.0.1:51010}
[junit4] 2> 1385943 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.e.j.s.Server Started @1388929ms
[junit4] 2> 1385943 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_EBA0DE45B66DBB6-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/_rbk/f, hostPort=51010, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_EBA0DE45B66DBB6-001/shard-1-001/cores}
[junit4] 2> 1385944 ERROR (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1385944 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 1385944 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1385944 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1385944 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-20T03:42:26.371Z
[junit4] 2> 1385949 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1385949 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_EBA0DE45B66DBB6-001/shard-1-001/solr.xml
[junit4] 2> 1385957 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48552/solr
[junit4] 2> 1385965 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [n:127.0.0.1:51010__rbk%2Ff ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1385967 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [n:127.0.0.1:51010__rbk%2Ff ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1385969 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [n:127.0.0.1:51010__rbk%2Ff ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51010__rbk%2Ff
[junit4] 2> 1385971 INFO (zkCallback-2105-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1385971 INFO (zkCallback-2101-thread-1-processing-n:127.0.0.1:44327__rbk%2Ff) [n:127.0.0.1:44327__rbk%2Ff ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1385971 INFO (zkCallback-2110-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1386069 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [n:127.0.0.1:51010__rbk%2Ff ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_EBA0DE45B66DBB6-001/shard-1-001/cores
[junit4] 2> 1386069 INFO (TEST-ShardSplitTest.testSplitShardWithRule-seed#[EBA0DE45B66DBB6]) [n:127.0.0.1:51010__rbk%2Ff ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1386071 INFO (OverseerStateUpdate-97647417391972356-127.0.0.1:44327__rbk%2Ff-n_0000000000) [n:127.0.0.1:44327__rbk%2Ff ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
[junit4] 2> 1387082 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1387094 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1387166 WARN (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1387167 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1387177 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 1387178 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1387178 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_EBA0DE45B66DBB6-001/shard-1-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_EBA0DE45B66DBB6-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 1387178 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@97ad257
[junit4] 2> 1387180 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=22, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 1387187 WARN (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1387222 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1387222 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1387223 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1387223 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1387224 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=36, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 1387224 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5216e03c[collection1] main]
[junit4] 2> 1387225 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1387225 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1387225 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1387226 INFO (searcherExecutor-5985-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff x:collection1 c:collection1) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5216e03c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1387226 INFO (coreLoadExecutor-5984-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff) [n:127.0.0.1:51010__rbk%2Ff c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1562358681596592128
[junit4] 2> 1387231 INFO (coreZkRegister-5979-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff x:collection1 c:collection1) [n:127.0.0.1:51010__rbk%2Ff c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1387231 INFO (coreZkRegister-5979-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff x:collection1 c:collection1) [n:127.0.0.1:51010__rbk%2Ff c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1387231 INFO (coreZkRegister-5979-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff x:collection1 c:collection1) [n:127.0.0.1:51010__rbk%2Ff c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:51010/_rbk/f/collection1/
[junit4] 2> 1387231 INFO (coreZkRegister-5979-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff x:collection1 c:collection1) [n:127.0.0.1:51010__rbk%2Ff c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1387231 INFO (coreZkRegister-5979-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff x:collection1 c:collection1) [n:127.0.0.1:51010__rbk%2Ff c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:51010/_rbk/f/collection1/ has no replicas
[junit4] 2> 1387232 INFO (coreZkRegister-5979-thread-1-processing-n:127.0.0.1:51010__rbk%2Ff x:collection1 c:collection1) [n:127.0.0.1:51010__rbk%2Ff c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLe
[...truncated too long message...]
ode4 x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@4a08df61
[junit4] 2> 2078392 INFO (coreCloseExecutor-6641-thread-1) [n:127.0.0.1:49221__rbk%2Ff c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1.shard1.core_node4, tag=1242095457
[junit4] 2> 2078393 INFO (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.Overseer Overseer (id=97647432133705746-127.0.0.1:49221__rbk%2Ff-n_0000000004) closing
[junit4] 2> 2078393 INFO (OverseerStateUpdate-97647432133705746-127.0.0.1:49221__rbk%2Ff-n_0000000004) [n:127.0.0.1:49221__rbk%2Ff ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:49221__rbk%2Ff
[junit4] 2> 2079895 WARN (zkCallback-2312-thread-8-processing-n:127.0.0.1:49221__rbk%2Ff) [n:127.0.0.1:49221__rbk%2Ff ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 2079896 INFO (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[EBA0DE45B66DBB6]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@5d3f0ad6{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 2079896 INFO (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[EBA0DE45B66DBB6]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@611b21ff{/_rbk/f,null,UNAVAILABLE}
[junit4] 2> 2079898 INFO (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[EBA0DE45B66DBB6]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:56990 56990
[junit4] 2> 2079998 INFO (Thread-5473) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:56990 56990
[junit4] 2> 2079999 WARN (Thread-5473) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/aliases.json
[junit4] 2> 7 /solr/clusterprops.json
[junit4] 2> 6 /solr/security.json
[junit4] 2> 6 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/clusterstate.json
[junit4] 2> 6 /solr/collections/collection1/state.json
[junit4] 2> 2 /solr/overseer_elect/election/97647432133705737-127.0.0.1:33462__rbk%2Ff-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 140 /solr/overseer/collection-queue-work
[junit4] 2> 57 /solr/overseer/queue
[junit4] 2> 18 /solr/overseer/queue-work
[junit4] 2> 7 /solr/collections
[junit4] 2> 6 /solr/live_nodes
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ShardSplitTest -Dtests.method=testSplitWithChaosMonkey -Dtests.seed=EBA0DE45B66DBB6 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es-NI -Dtests.timezone=Australia/Melbourne -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 471s J2 | ShardSplitTest.testSplitWithChaosMonkey <<<
[junit4] > Throwable #1: java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
[junit4] > at __randomizedtesting.SeedInfo.seed([EBA0DE45B66DBB6:859DDE351A607032]:0)
[junit4] > at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:187)
[junit4] > at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:144)
[junit4] > at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
[junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:870)
[junit4] > at org.apache.solr.cloud.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:442)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2080000 INFO (SUITE-ShardSplitTest-seed#[EBA0DE45B66DBB6]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.ShardSplitTest_EBA0DE45B66DBB6-001
[junit4] 2> Mar 20, 2017 3:54:00 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {rnd_b=Lucene50(blocksize=128), shard_s=PostingsFormat(name=MockRandom), n_ti=PostingsFormat(name=MockRandom), a_t=PostingsFormat(name=MockRandom), a_i=Lucene50(blocksize=128), id=Lucene50(blocksize=128)}, docValues:{rnd_b=DocValuesFormat(name=Direct), shard_s=DocValuesFormat(name=Lucene70), _version_=DocValuesFormat(name=Lucene70), n_ti=DocValuesFormat(name=Lucene70), a_t=DocValuesFormat(name=Lucene70), a_i=DocValuesFormat(name=Direct), id=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=1357, maxMBSortInHeap=5.857394070026441, sim=RandomSimilarity(queryNorm=true): {}, locale=es-NI, timezone=Australia/Melbourne
[junit4] 2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=4,threads=1,free=132042160,total=529006592
[junit4] 2> NOTE: All tests run in this JVM: [TestReRankQParserPlugin, DeleteLastCustomShardedReplicaTest, TestConfigsApi, SuggestComponentContextFilterQueryTest, TestGroupingSearch, BlockCacheTest, TestFieldCacheSortRandom, SimpleFacetsTest, SolrCLIZkUtilsTest, BinaryUpdateRequestHandlerTest, TestIndexSearcher, TestZkChroot, TestUniqueKeyFieldResource, TestConfig, TestSolrDeletionPolicy1, HLLUtilTest, DistributedFacetPivotLongTailTest, TestSolr4Spatial2, PrimitiveFieldTypeTest, TestRestManager, DistributedSuggestComponentTest, TestDistributedMissingSort, TestIndexingPerformance, TestSegmentSorting, TestConfigSets, HighlighterConfigTest, RecoveryZkTest, PreAnalyzedFieldTest, JSONWriterTest, TestManagedSchemaAPI, OutputWriterTest, TestUseDocValuesAsStored2, HdfsRecoveryZkTest, StatsComponentTest, AutoCommitTest, TriLevelCompositeIdRoutingTest, ShowFileRequestHandlerTest, CdcrVersionReplicationTest, TestStressCloudBlindAtomicUpdates, BadIndexSchemaTest, TestComponentsName, TestStandardQParsers, TestMiniSolrCloudClusterSSL, ConcurrentDeleteAndCreateCollectionTest, TestCloudSchemaless, TestLeaderElectionZkExpiry, CdcrReplicationDistributedZkTest, BadCopyFieldTest, TestSolrXml, LeaderElectionTest, TestChildDocTransformer, RuleEngineTest, TestCloudPivotFacet, TestDeleteCollectionOnDownNodes, TestDocSet, TestTolerantUpdateProcessorCloud, TestCSVResponseWriter, TestFastOutputStream, MBeansHandlerTest, CheckHdfsIndexTest, V2ApiIntegrationTest, SolrPluginUtilsTest, SolrCoreMetricManagerTest, SuggesterTSTTest, HdfsCollectionsAPIDistributedZkTest, UpdateLogTest, DebugComponentTest, HardAutoCommitTest, CoreMergeIndexesAdminHandlerTest, TermVectorComponentDistributedTest, BlockJoinFacetRandomTest, HdfsThreadLeakTest, SolrTestCaseJ4Test, StatelessScriptUpdateProcessorFactoryTest, ResourceLoaderTest, SliceStateTest, DocValuesMultiTest, EchoParamsTest, TestSolrCloudWithHadoopAuthPlugin, ManagedSchemaRoundRobinCloudTest, DirectUpdateHandlerTest, ReplicationFactorTest, CreateCollectionCleanupTest, MigrateRouteKeyTest, TestCollationField, XmlUpdateRequestHandlerTest, SharedFSAutoReplicaFailoverTest, TestDynamicFieldResource, TestExactSharedStatsCache, StatsReloadRaceTest, TestUninvertingReader, TestClusterProperties, HdfsDirectoryFactoryTest, DistributedFacetPivotWhiteBoxTest, CursorMarkTest, TestSolrQueryParser, TestMultiWordSynonyms, TestSortByMinMaxFunction, TestLockTree, TestRecovery, TestQueryWrapperFilter, BasicDistributedZk2Test, HttpPartitionTest, FullSolrCloudDistribCmdsTest, TestHdfsBackupRestoreCore, TestSolrCloudWithDelegationTokens, TestRequestForwarding, TestCoreDiscovery, SynonymTokenizerTest, TestScoreJoinQPNoScore, BasicAuthIntegrationTest, TestDFISimilarityFactory, LeaderElectionIntegrationTest, QueryElevationComponentTest, TestLazyCores, TestBadConfig, HighlighterTest, TestExtendedDismaxParser, SolrRequestParserTest, TestFoldingMultitermQuery, SuggesterTest, TestStressLucene, TestCSVLoader, SchemaVersionSpecificBehaviorTest, TestUpdate, FieldMutatingUpdateProcessorTest, DirectUpdateHandlerOptimizeTest, SolrInfoMBeanTest, IndexBasedSpellCheckerTest, DisMaxRequestHandlerTest, TestQueryUtils, TestWriterPerf, DocumentAnalysisRequestHandlerTest, TestOmitPositions, TestValueSourceCache, FieldAnalysisRequestHandlerTest, ReturnFieldsTest, TestPHPSerializedResponseWriter, ResponseLogComponentTest, SolrIndexConfigTest, MultiTermTest, SampleTest, TestBinaryField, NumericFieldsTest, DirectSolrConnectionTest, TestLuceneMatchVersion, TestSweetSpotSimilarityFactory, TestLMDirichletSimilarityFactory, TestIBSimilarityFactory, TestJmxMonitoredMap, URLClassifyProcessorTest, PrimUtilsTest, DateFieldTest, ClusterStateTest, SystemInfoHandlerTest, CircularListTest, DistributedIntervalFacetingTest, AliasIntegrationTest, CdcrReplicationHandlerTest, CdcrRequestHandlerTest, CloudExitableDirectoryReaderTest, CollectionStateFormat2Test, CollectionsAPIAsyncDistributedZkTest, DeleteInactiveReplicaTest, DeleteNodeTest, DistribCursorPagingTest, LeaderFailoverAfterPartitionTest, MissingSegmentRecoveryTest, OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, OverseerRolesTest, OverseerStatusTest, RecoveryAfterSoftCommitTest, SSLMigrationTest, ShardSplitTest]
[junit4] Completed [675/701 (2!)] on J2 in 696.72s, 5 tests, 1 failure <<< FAILURES!
[...truncated 62715 lines...]