You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/01/24 17:22:48 UTC
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3795
- Still Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3795/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
3 tests failed.
FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI
Error Message:
Error from server at https://127.0.0.1:64100/solr/awhollynewcollection_0: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 510 </title> </head> <body> <h2>HTTP ERROR: 510</h2> <p>Problem accessing /solr/awhollynewcollection_0/select. Reason: <pre> {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg={"awhollynewcollection_0":4},code=510}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/> </body> </html>
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:64100/solr/awhollynewcollection_0: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 510 </title>
</head>
<body>
<h2>HTTP ERROR: 510</h2>
<p>Problem accessing /solr/awhollynewcollection_0/select. Reason:
<pre> {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg={"awhollynewcollection_0":4},code=510}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/>
</body>
</html>
at __randomizedtesting.SeedInfo.seed([C8AE32E10101EF03:80DB46550732C096]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:595)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:279)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:439)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:391)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1344)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1095)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1198)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1198)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1198)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1198)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1198)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1037)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:517)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
FAILED: org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigAliasReplication
Error Message:
expected:<1> but was:<0>
Stack Trace:
java.lang.AssertionError: expected:<1> but was:<0>
at __randomizedtesting.SeedInfo.seed([C8AE32E10101EF03:3FDDDCB9C7E940E5]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigAliasReplication(TestReplicationHandler.java:1345)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
FAILED: org.apache.solr.cloud.PeerSyncReplicationTest.test
Error Message:
timeout waiting to see all nodes active
Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
at __randomizedtesting.SeedInfo.seed([C8AE32E10101EF03:40FA0D3BAFFD82FB]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 11191 lines...]
[junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/init-core-data-001
[junit4] 2> 975306 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[C8AE32E10101EF03]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields
[junit4] 2> 975311 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[C8AE32E10101EF03]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 975312 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[C8AE32E10101EF03]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-001
[junit4] 2> 975312 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[C8AE32E10101EF03]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 975313 INFO (Thread-1591) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 975313 INFO (Thread-1591) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 975418 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[C8AE32E10101EF03]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:64098
[junit4] 2> 975438 INFO (jetty-launcher-1177-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 975439 INFO (jetty-launcher-1177-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 975440 INFO (jetty-launcher-1177-thread-3) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 975441 INFO (jetty-launcher-1177-thread-4) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 975444 INFO (jetty-launcher-1177-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@23f6fcad{/solr,null,AVAILABLE}
[junit4] 2> 975444 INFO (jetty-launcher-1177-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@550508cf{/solr,null,AVAILABLE}
[junit4] 2> 975445 INFO (jetty-launcher-1177-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5d1bd7e9{/solr,null,AVAILABLE}
[junit4] 2> 975445 INFO (jetty-launcher-1177-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@53ccbec0{SSL,[ssl, http/1.1]}{127.0.0.1:64101}
[junit4] 2> 975445 INFO (jetty-launcher-1177-thread-2) [ ] o.e.j.s.Server Started @981444ms
[junit4] 2> 975445 INFO (jetty-launcher-1177-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=64101}
[junit4] 2> 975445 INFO (jetty-launcher-1177-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1bcbea53{SSL,[ssl, http/1.1]}{127.0.0.1:64100}
[junit4] 2> 975445 INFO (jetty-launcher-1177-thread-1) [ ] o.e.j.s.Server Started @981444ms
[junit4] 2> 975445 INFO (jetty-launcher-1177-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=64100}
[junit4] 2> 975445 ERROR (jetty-launcher-1177-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 975445 ERROR (jetty-launcher-1177-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 975446 INFO (jetty-launcher-1177-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 975446 INFO (jetty-launcher-1177-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 975446 INFO (jetty-launcher-1177-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 975446 INFO (jetty-launcher-1177-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 975446 INFO (jetty-launcher-1177-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 975446 INFO (jetty-launcher-1177-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 975446 INFO (jetty-launcher-1177-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-24T16:16:26.028Z
[junit4] 2> 975446 INFO (jetty-launcher-1177-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-24T16:16:26.028Z
[junit4] 2> 975446 INFO (jetty-launcher-1177-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@60feb5d4{/solr,null,AVAILABLE}
[junit4] 2> 975447 INFO (jetty-launcher-1177-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5e835f03{SSL,[ssl, http/1.1]}{127.0.0.1:64102}
[junit4] 2> 975447 INFO (jetty-launcher-1177-thread-4) [ ] o.e.j.s.Server Started @981446ms
[junit4] 2> 975447 INFO (jetty-launcher-1177-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=64102}
[junit4] 2> 975447 ERROR (jetty-launcher-1177-thread-4) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 975447 INFO (jetty-launcher-1177-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2d046058{SSL,[ssl, http/1.1]}{127.0.0.1:64103}
[junit4] 2> 975447 INFO (jetty-launcher-1177-thread-3) [ ] o.e.j.s.Server Started @981447ms
[junit4] 2> 975447 INFO (jetty-launcher-1177-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=64103}
[junit4] 2> 975448 INFO (jetty-launcher-1177-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 975449 INFO (jetty-launcher-1177-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 975449 INFO (jetty-launcher-1177-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 975449 INFO (jetty-launcher-1177-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-24T16:16:26.031Z
[junit4] 2> 975449 ERROR (jetty-launcher-1177-thread-3) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 975450 INFO (jetty-launcher-1177-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 975450 INFO (jetty-launcher-1177-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 975450 INFO (jetty-launcher-1177-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 975450 INFO (jetty-launcher-1177-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-24T16:16:26.032Z
[junit4] 2> 975453 INFO (jetty-launcher-1177-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 975453 INFO (jetty-launcher-1177-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 975453 INFO (jetty-launcher-1177-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 975454 INFO (jetty-launcher-1177-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 975467 INFO (jetty-launcher-1177-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:64098/solr
[junit4] 2> 975467 INFO (jetty-launcher-1177-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:64098/solr
[junit4] 2> 975469 INFO (jetty-launcher-1177-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:64098/solr
[junit4] 2> 975470 INFO (jetty-launcher-1177-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:64098/solr
[junit4] 2> 975524 INFO (jetty-launcher-1177-thread-4) [n:127.0.0.1:64102_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:64102_solr
[junit4] 2> 975525 INFO (jetty-launcher-1177-thread-3) [n:127.0.0.1:64103_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64103_solr
[junit4] 2> 975525 INFO (jetty-launcher-1177-thread-1) [n:127.0.0.1:64100_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64100_solr
[junit4] 2> 975525 INFO (jetty-launcher-1177-thread-2) [n:127.0.0.1:64101_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64101_solr
[junit4] 2> 975528 INFO (zkCallback-1196-thread-1-processing-n:127.0.0.1:64101_solr) [n:127.0.0.1:64101_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 975529 INFO (zkCallback-1197-thread-1-processing-n:127.0.0.1:64102_solr) [n:127.0.0.1:64102_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 975530 INFO (zkCallback-1198-thread-1-processing-n:127.0.0.1:64103_solr) [n:127.0.0.1:64103_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 975530 INFO (zkCallback-1195-thread-1-processing-n:127.0.0.1:64100_solr) [n:127.0.0.1:64100_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 975531 INFO (jetty-launcher-1177-thread-4) [n:127.0.0.1:64102_solr ] o.a.s.c.Overseer Overseer (id=97338955261345803-127.0.0.1:64102_solr-n_0000000000) starting
[junit4] 2> 975534 INFO (zkCallback-1198-thread-1-processing-n:127.0.0.1:64103_solr) [n:127.0.0.1:64103_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 975534 INFO (zkCallback-1196-thread-1-processing-n:127.0.0.1:64101_solr) [n:127.0.0.1:64101_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 975535 INFO (zkCallback-1195-thread-1-processing-n:127.0.0.1:64100_solr) [n:127.0.0.1:64100_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 975535 INFO (zkCallback-1197-thread-1-processing-n:127.0.0.1:64102_solr) [n:127.0.0.1:64102_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 975557 INFO (jetty-launcher-1177-thread-4) [n:127.0.0.1:64102_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64102_solr
[junit4] 2> 975560 INFO (zkCallback-1198-thread-1-processing-n:127.0.0.1:64103_solr) [n:127.0.0.1:64103_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 975560 INFO (zkCallback-1197-thread-1-processing-n:127.0.0.1:64102_solr) [n:127.0.0.1:64102_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 975560 INFO (zkCallback-1196-thread-1-processing-n:127.0.0.1:64101_solr) [n:127.0.0.1:64101_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 975561 INFO (zkCallback-1195-thread-1-processing-n:127.0.0.1:64100_solr) [n:127.0.0.1:64100_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 975612 INFO (jetty-launcher-1177-thread-3) [n:127.0.0.1:64103_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-001/node3/.
[junit4] 2> 975645 INFO (jetty-launcher-1177-thread-1) [n:127.0.0.1:64100_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-001/node1/.
[junit4] 2> 975651 INFO (jetty-launcher-1177-thread-2) [n:127.0.0.1:64101_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-001/node2/.
[junit4] 2> 975705 INFO (jetty-launcher-1177-thread-4) [n:127.0.0.1:64102_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-001/node4/.
[junit4] 2> 975966 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[C8AE32E10101EF03]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 975968 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[C8AE32E10101EF03]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64098/solr ready
[junit4] 2> 976012 INFO (TEST-CollectionsAPIDistributedZkTest.testNoCollectionSpecified-seed#[C8AE32E10101EF03]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNoCollectionSpecified
[junit4] 2> 976013 INFO (TEST-CollectionsAPIDistributedZkTest.testNoCollectionSpecified-seed#[C8AE32E10101EF03]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 976037 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr ] o.a.s.h.a.CoreAdminOperation core create command dataDir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-002&collection.configName=conf&name=corewithnocollection&action=CREATE&numShards=1&collection=&wt=javabin&version=2
[junit4] 2> 976041 INFO (OverseerStateUpdate-97338955261345803-127.0.0.1:64102_solr-n_0000000000) [n:127.0.0.1:64102_solr ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 977060 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 977073 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.s.IndexSchema [corewithnocollection] Schema name=minimal
[junit4] 2> 977075 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 977076 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.CoreContainer Creating SolrCore 'corewithnocollection' using configuration from collection corewithnocollection
[junit4] 2> 977077 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.SolrCore [[corewithnocollection] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-001/node3/corewithnocollection], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-002/]
[junit4] 2> 977172 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 977172 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 977174 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 977174 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 977197 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.s.SolrIndexSearcher Opening [Searcher@597b102c[corewithnocollection] main]
[junit4] 2> 977199 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 977200 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 977200 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 977200 INFO (searcherExecutor-3678-thread-1-processing-n:127.0.0.1:64103_solr x:corewithnocollection c:corewithnocollection) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.SolrCore [corewithnocollection] Registered new searcher Searcher@597b102c[corewithnocollection] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 977201 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557423286159147008
[junit4] 2> 977216 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 977216 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 977216 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:64103/solr/corewithnocollection/
[junit4] 2> 977216 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 977217 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.SyncStrategy https://127.0.0.1:64103/solr/corewithnocollection/ has no replicas
[junit4] 2> 977231 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:64103/solr/corewithnocollection/ shard1
[junit4] 2> 977341 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 977353 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection x:corewithnocollection] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={dataDir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-002&collection.configName=conf&name=corewithnocollection&action=CREATE&numShards=1&collection=&wt=javabin&version=2} status=0 QTime=1316
[junit4] 2> 977357 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr ] o.a.s.h.a.CoreAdminOperation core create command dataDir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-002&collection.configName=conf&name=corewithnocollection2&action=CREATE&numShards=1&wt=javabin&version=2
[junit4] 2> 977361 INFO (OverseerStateUpdate-97338955261345803-127.0.0.1:64102_solr-n_0000000000) [n:127.0.0.1:64102_solr ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 978373 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 978390 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.s.IndexSchema [corewithnocollection2] Schema name=minimal
[junit4] 2> 978392 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 978393 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.CoreContainer Creating SolrCore 'corewithnocollection2' using configuration from collection corewithnocollection2
[junit4] 2> 978394 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.SolrCore [[corewithnocollection2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-001/node3/corewithnocollection2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-002/]
[junit4] 2> 978510 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 978510 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 978511 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 978511 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 978535 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@3e7ac215[corewithnocollection2] main]
[junit4] 2> 978537 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 978538 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 978538 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 978539 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557423287562141696
[junit4] 2> 978539 INFO (searcherExecutor-3683-thread-1-processing-n:127.0.0.1:64103_solr x:corewithnocollection2 c:corewithnocollection2) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.SolrCore [corewithnocollection2] Registered new searcher Searcher@3e7ac215[corewithnocollection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 978554 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 978554 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 978554 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:64103/solr/corewithnocollection2/
[junit4] 2> 978554 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 978555 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.SyncStrategy https://127.0.0.1:64103/solr/corewithnocollection2/ has no replicas
[junit4] 2> 978564 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:64103/solr/corewithnocollection2/ shard1
[junit4] 2> 978721 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 978728 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={dataDir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-002&collection.configName=conf&name=corewithnocollection2&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=1371
[junit4] 2> 978730 INFO (TEST-CollectionsAPIDistributedZkTest.testNoCollectionSpecified-seed#[C8AE32E10101EF03]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNoCollectionSpecified
[junit4] 2> 978768 INFO (TEST-CollectionsAPIDistributedZkTest.testSpecificConfigsets-seed#[C8AE32E10101EF03]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSpecificConfigsets
[junit4] 2> 978770 INFO (TEST-CollectionsAPIDistributedZkTest.testSpecificConfigsets-seed#[C8AE32E10101EF03]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 978773 INFO (qtp1217080757-7094) [n:127.0.0.1:64103_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=corewithnocollection&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 978777 INFO (OverseerThreadFactory-3669-thread-1-processing-n:127.0.0.1:64102_solr) [n:127.0.0.1:64102_solr ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
[junit4] 2> 978783 INFO (qtp1217080757-7074) [n:127.0.0.1:64103_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.corewithnocollection
[junit4] 2> 978784 INFO (qtp1217080757-7074) [n:127.0.0.1:64103_solr ] o.a.s.c.SolrCore [corewithnocollection] CLOSING SolrCore org.apache.solr.core.SolrCore@46babeeb
[junit4] 2> 978795 INFO (qtp1217080757-7074) [n:127.0.0.1:64103_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.corewithnocollection
[junit4] 2> 978811 INFO (qtp1217080757-7074) [n:127.0.0.1:64103_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=corewithnocollection&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=29
[junit4] 2> 979535 INFO (qtp1217080757-7094) [n:127.0.0.1:64103_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=corewithnocollection&action=DELETE&wt=javabin&version=2} status=0 QTime=762
[junit4] 2> 979537 INFO (qtp1217080757-7088) [n:127.0.0.1:64103_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=corewithnocollection2&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 979541 INFO (OverseerThreadFactory-3669-thread-2-processing-n:127.0.0.1:64102_solr) [n:127.0.0.1:64102_solr ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
[junit4] 2> 979542 INFO (OverseerCollectionConfigSetProcessor-97338955261345803-127.0.0.1:64102_solr-n_0000000000) [n:127.0.0.1:64102_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 979544 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.corewithnocollection2
[junit4] 2> 979544 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr ] o.a.s.c.SolrCore [corewithnocollection2] CLOSING SolrCore org.apache.solr.core.SolrCore@6f52614e
[junit4] 2> 979556 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.corewithnocollection2
[junit4] 2> 979569 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=corewithnocollection2&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=26
[junit4] 2> 980281 INFO (qtp1217080757-7088) [n:127.0.0.1:64103_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=corewithnocollection2&action=DELETE&wt=javabin&version=2} status=0 QTime=745
[junit4] 2> 980283 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf2&name=withconfigset2&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 980287 INFO (OverseerThreadFactory-3669-thread-3-processing-n:127.0.0.1:64102_solr) [n:127.0.0.1:64102_solr ] o.a.s.c.CreateCollectionCmd Create collection withconfigset2
[junit4] 2> 980289 INFO (OverseerCollectionConfigSetProcessor-97338955261345803-127.0.0.1:64102_solr-n_0000000000) [n:127.0.0.1:64102_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 980534 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf2&newCollection=true&name=withconfigset2_shard1_replica1&action=CREATE&numShards=1&collection=withconfigset2&shard=shard1&wt=javabin&version=2
[junit4] 2> 980651 INFO (zkCallback-1196-thread-2-processing-n:127.0.0.1:64101_solr) [n:127.0.0.1:64101_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/withconfigset2/state.json] for collection [withconfigset2] has occurred - updating... (live nodes size: [4])
[junit4] 2> 981562 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 981584 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.s.IndexSchema [withconfigset2_shard1_replica1] Schema name=minimal
[junit4] 2> 981587 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 981589 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'withconfigset2_shard1_replica1' using configuration from collection withconfigset2
[junit4] 2> 981590 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.c.SolrCore [[withconfigset2_shard1_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-001/node2/withconfigset2_shard1_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-001/node2/./withconfigset2_shard1_replica1/data/]
[junit4] 2> 981590 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@658371d2
[junit4] 2> 981615 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 981615 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 981617 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 981617 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 981618 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@458d868d[withconfigset2_shard1_replica1] main]
[junit4] 2> 981619 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf2
[junit4] 2> 981620 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf2
[junit4] 2> 981620 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 981621 INFO (searcherExecutor-3688-thread-1-processing-n:127.0.0.1:64101_solr x:withconfigset2_shard1_replica1 s:shard1 c:withconfigset2) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.c.SolrCore [withconfigset2_shard1_replica1] Registered new searcher Searcher@458d868d[withconfigset2_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 981622 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557423290794901504
[junit4] 2> 981632 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 981632 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 981632 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:64101/solr/withconfigset2_shard1_replica1/
[junit4] 2> 981632 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 981632 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.c.SyncStrategy https://127.0.0.1:64101/solr/withconfigset2_shard1_replica1/ has no replicas
[junit4] 2> 981640 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:64101/solr/withconfigset2_shard1_replica1/ shard1
[junit4] 2> 981756 INFO (zkCallback-1196-thread-2-processing-n:127.0.0.1:64101_solr) [n:127.0.0.1:64101_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/withconfigset2/state.json] for collection [withconfigset2] has occurred - updating... (live nodes size: [4])
[junit4] 2> 981809 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 981813 INFO (qtp2076956430-7077) [n:127.0.0.1:64101_solr c:withconfigset2 s:shard1 x:withconfigset2_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf2&newCollection=true&name=withconfigset2_shard1_replica1&action=CREATE&numShards=1&collection=withconfigset2&shard=shard1&wt=javabin&version=2} status=0 QTime=1279
[junit4] 2> 981817 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 981927 INFO (zkCallback-1196-thread-2-processing-n:127.0.0.1:64101_solr) [n:127.0.0.1:64101_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/withconfigset2/state.json] for collection [withconfigset2] has occurred - updating... (live nodes size: [4])
[junit4] 2> 982303 INFO (OverseerCollectionConfigSetProcessor-97338955261345803-127.0.0.1:64102_solr-n_0000000000) [n:127.0.0.1:64102_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 982829 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf2&name=withconfigset2&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2546
[junit4] 2> 982831 INFO (TEST-CollectionsAPIDistributedZkTest.testSpecificConfigsets-seed#[C8AE32E10101EF03]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSpecificConfigsets
[junit4] 2> 982874 INFO (TEST-CollectionsAPIDistributedZkTest.deleteCollectionRemovesStaleZkCollectionsNode-seed#[C8AE32E10101EF03]) [ ] o.a.s.SolrTestCaseJ4 ###Starting deleteCollectionRemovesStaleZkCollectionsNode
[junit4] 2> 982876 INFO (TEST-CollectionsAPIDistributedZkTest.deleteCollectionRemovesStaleZkCollectionsNode-seed#[C8AE32E10101EF03]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 982879 INFO (qtp1217080757-7095) [n:127.0.0.1:64103_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=withconfigset2&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 982883 INFO (OverseerThreadFactory-3669-thread-4-processing-n:127.0.0.1:64102_solr) [n:127.0.0.1:64102_solr ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
[junit4] 2> 982888 INFO (qtp2076956430-7079) [n:127.0.0.1:64101_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.withconfigset2_shard1_replica1
[junit4] 2> 982889 INFO (qtp2076956430-7079) [n:127.0.0.1:64101_solr ] o.a.s.c.SolrCore [withconfigset2_shard1_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@297e697d
[junit4] 2> 982931 INFO (qtp2076956430-7079) [n:127.0.0.1:64101_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.withconfigset2.shard1.replica1
[junit4] 2> 982935 INFO (qtp2076956430-7079) [n:127.0.0.1:64101_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=withconfigset2_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=47
[junit4] 2> 983673 INFO (qtp1217080757-7095) [n:127.0.0.1:64103_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=withconfigset2&action=DELETE&wt=javabin&version=2} status=0 QTime=794
[junit4] 2> 983679 INFO (qtp1217080757-7066) [n:127.0.0.1:64103_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=out_of_sync_collection&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 983684 INFO (OverseerCollectionConfigSetProcessor-97338955261345803-127.0.0.1:64102_solr-n_0000000000) [n:127.0.0.1:64102_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 983689 INFO (qtp1217080757-7066) [n:127.0.0.1:64103_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=out_of_sync_collection&action=DELETE&wt=javabin&version=2} status=0 QTime=10
[junit4] 2> 983690 INFO (qtp1217080757-7074) [n:127.0.0.1:64103_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 983690 INFO (qtp1217080757-7074) [n:127.0.0.1:64103_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 983691 INFO (TEST-CollectionsAPIDistributedZkTest.deleteCollectionRemovesStaleZkCollectionsNode-seed#[C8AE32E10101EF03]) [ ] o.a.s.SolrTestCaseJ4 ###Ending deleteCollectionRemovesStaleZkCollectionsNode
[junit4] 2> 983731 INFO (TEST-CollectionsAPIDistributedZkTest.testBadActionNames-seed#[C8AE32E10101EF03]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testBadActionNames
[junit4] 2> 983732 INFO (TEST-CollectionsAPIDistributedZkTest.testBadActionNames-seed#[C8AE32E10101EF03]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 983735 ERROR (qtp1217080757-7094) [n:127.0.0.1:64103_solr ] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Unknown action: BADACTION
[junit4] 2> at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:203)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:166)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:664)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:445)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:296)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:113)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
[junit4] 2> at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 983736 INFO (qtp1217080757-7094) [n:127.0.0.1:64103_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=badactioncollection&action=BADACTION&numShards=2&wt=javabin&version=2} status=400 QTime=0
[junit4] 2> 983736 INFO (TEST-CollectionsAPIDistributedZkTest.testBadActionNames-seed#[C8AE32E10101EF03]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testBadActionNames
[junit4] 2> 983769 INFO (TEST-CollectionsAPIDistributedZkTest.testNoConfigSetExist-seed#[C8AE32E10101EF03]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNoConfigSetExist
[junit4] 2> 983771 INFO (TEST-CollectionsAPIDistributedZkTest.testNoConfigSetExist-seed#[C8AE32E10101EF03]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 983774 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr ] o.a.s.h.a.CoreAdminOperation core create command dataDir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-003&collection.configName=conf123&name=corewithnocollection3&action=CREATE&numShards=1&collection=&wt=javabin&version=2
[junit4] 2> 983777 INFO (OverseerStateUpdate-97338955261345803-127.0.0.1:64102_solr-n_0000000000) [n:127.0.0.1:64102_solr ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 984786 ERROR (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection3 x:corewithnocollection3] o.a.s.c.c.ZkStateReader Specified config=[conf123] does not exist in ZooKeeper at location=[/configs/conf123]
[junit4] 2> 984788 ERROR (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection3 x:corewithnocollection3] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'corewithnocollection3': Unable to create core [corewithnocollection3] Caused by: Specified config does not exist in ZooKeeper: conf123
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:897)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:88)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:378)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:379)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:165)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:166)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:664)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:445)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:296)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:113)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
[junit4] 2> at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> Caused by: org.apache.solr.common.SolrException: Unable to create core [corewithnocollection3]
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:942)
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:863)
[junit4] 2> ... 36 more
[junit4] 2> Caused by: org.apache.solr.common.cloud.ZooKeeperException: Specified config does not exist in ZooKeeper: conf123
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.readConfigName(ZkStateReader.java:189)
[junit4] 2> at org.apache.solr.cloud.CloudConfigSetService.createCoreResourceLoader(CloudConfigSetService.java:54)
[junit4] 2> at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:924)
[junit4] 2> ... 37 more
[junit4] 2>
[junit4] 2> 984788 INFO (qtp1217080757-7089) [n:127.0.0.1:64103_solr c:corewithnocollection3 x:corewithnocollection3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={dataDir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-003&collection.configName=conf123&name=corewithnocollection3&action=CREATE&numShards=1&collection=&wt=javabin&version=2} status=400 QTime=1013
[junit4] 2> 984996 INFO (qtp1217080757-7088) [n:127.0.0.1:64103_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 984996 INFO (qtp1217080757-7088) [n:127.0.0.1:64103_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 984997 INFO (TEST-CollectionsAPIDistributedZkTest.testNoConfigSetExist-seed#[C8AE32E10101EF03]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNoConfigSetExist
[junit4] 2> 985031 INFO (TEST-CollectionsAPIDistributedZkTest.testMissingRequiredParameters-seed#[C8AE32E10101EF03]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testMissingRequiredParameters
[junit4] 2> 985033 INFO (TEST-CollectionsAPIDistributedZkTest.testMissingRequiredParameters-seed#[C8AE32E10101EF03]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 985036 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=corewithnocollection3&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 985039 INFO (OverseerThreadFactory-3669-thread-5-processing-n:127.0.0.1:64102_solr) [n:127.0.0.1:64102_solr ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
[junit4] 2> 985040 INFO (OverseerCollectionConfigSetProcessor-97338955261345803-127.0.0.1:64102_solr-n_0000000000) [n:127.0.0.1:64102_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000008 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 985767 INFO (qtp1217080757-7093) [n:127.0.0.1:64103_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=corewithnocollection3&action=DELETE&wt=javabin&version=2} status=0 QTime=731
[junit4] 2> 985769 INFO (qtp1217080757-7095) [n:127.0.0.1:64103_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 985769 ERROR (qtp1217080757-7095) [n:127.0.0.1:64103_solr ] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Missing required parameter: name
[junit4] 2> at org.apache.solr.common.params.RequiredSolrParams.getParams(RequiredSolrParams.java:90)
[junit4] 2> at org.apache.solr.common.params.SolrParams.getAll(SolrParams.java:461)
[junit4] 2> at org.apache.solr.common.params.SolrParams.getAll(SolrParams.java:478)
[junit4] 2> at org.apache.solr.handler.admin.CollectionsHandler$CollectionOperation.lambda$static$0(CollectionsHandler.java:383)
[junit4] 2> at org.apache.solr.handler.admin.CollectionsHandler$CollectionOperation.execute(CollectionsHandler.java:877)
[junit4] 2> at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:209)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:166)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:664)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:445)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:296)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:113)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
[junit4] 2> at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 985769 INFO (qtp1217080757-7095) [n:127.0.0.1:64103_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CREATE&numShards=2&wt=javabin&version=2} status=400 QTime=0
[junit4] 2> 985770 INFO (TEST-CollectionsAPIDistributedZkTest.testMissingRequiredParameters-seed#[C8AE32E10101EF03]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testMissingRequiredParameters
[junit4] 2> 985805 INFO (TEST-CollectionsAPIDistributedZkTest.addReplicaTest-seed#[C8AE32E10101EF03]) [ ] o.a.s.SolrTestCaseJ4 ###Starting addReplicaTest
[junit4] 2> 985807 INFO (TEST-CollectionsAPIDistributedZkTest.addReplicaTest-seed#[C8AE32E10101EF03]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 985809 INFO (qtp1217080757-7066) [n:127.0.0.1:64103_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=4&name=addReplicaColl&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 985812 INFO (OverseerThreadFactory-3669-thread-5-processing-n:127.0.0.1:64102_solr) [n:127.0.0.1:64102_solr ] o.a.s.c.CreateCollectionCmd Create collection addReplicaColl
[junit4] 2> 985813 INFO (OverseerCollectionConfigSetProcessor-97338955261345803-127.0.0.1:64102_solr-n_0000000000) [n:127.0.0.1:64102_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000010 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 986046 INFO (qtp2076956430-7080) [n:127.0.0.1:64101_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=addReplicaColl_shard1_replica2&action=CREATE&numShards=2&collection=addReplicaColl&shard=shard1&wt=javabin&version=2
[junit4] 2> 986053 INFO (qtp1217080757-7074) [n:127.0.0.1:64103_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=addReplicaColl_shard2_replica2&action=CREATE&numShards=2&collection=addReplicaColl&shard=shard2&wt=javabin&version=2
[junit4] 2> 986069 INFO (qtp1035706113-7067) [n:127.0.0.1:64100_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=addReplicaColl_shard1_replica1&action=CREATE&numShards=2&collection=addReplicaColl&shard=shard1&wt=javabin&version=2
[junit4] 2> 986074 INFO (qtp1913120776-7085) [n:127.0.0.1:64102_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=addReplicaColl_shard2_replica1&action=CREATE&numShards=2&collection=addReplicaColl&shard=shard2&wt=javabin&version=2
[junit4] 2> 986188 INFO (zkCallback-1197-thread-2-processing-n:127.0.0.1:64102_solr) [n:127.0.0.1:64102_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
[junit4] 2> 986188 INFO (zkCallback-1198-thread-1-processing-n:127.0.0.1:64103_solr) [n:127.0.0.1:64103_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
[junit4] 2> 986189 INFO (zkCallback-1195-thread-1-processing-n:127.0.0.1:64100_solr) [n:127.0.0.1:64100_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
[junit4] 2> 986188 INFO (zkCallback-1196-thread-2-processing-n:127.0.0.1:64101_solr) [n:127.0.0.1:64101_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
[junit4] 2> 987076 INFO (qtp2076956430-7080) [n:127.0.0.1:64101_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 987085 INFO (qtp2076956430-7080) [n:127.0.0.1:64101_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.s.IndexSchema [addReplicaColl_shard1_replica2] Schema name=minimal
[junit4] 2> 987088 INFO (qtp2076956430-7080) [n:127.0.0.1:64101_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 987093 INFO (qtp2076956430-7080) [n:127.0.0.1:64101_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'addReplicaColl_shard1_replica2' using configuration from collection addReplicaColl
[junit4] 2> 987093 INFO (qtp2076956430-7080) [n:127.0.0.1:64101_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.c.SolrCore [[addReplicaColl_shard1_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-001/node2/addReplicaColl_shard1_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-001/node2/./addReplicaColl_shard1_replica2/data/]
[junit4] 2> 987098 INFO (qtp1217080757-7074) [n:127.0.0.1:64103_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 987112 INFO (qtp1913120776-7085) [n:127.0.0.1:64102_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 987114 INFO (qtp1035706113-7067) [n:127.0.0.1:64100_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 987131 INFO (qtp1913120776-7085) [n:127.0.0.1:64102_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica1] o.a.s.s.IndexSchema [addReplicaColl_shard2_replica1] Schema name=minimal
[junit4] 2> 987131 INFO (qtp1217080757-7074) [n:127.0.0.1:64103_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica2] o.a.s.s.IndexSchema [addReplicaColl_shard2_replica2] Schema name=minimal
[junit4] 2> 987132 INFO (qtp1913120776-7085) [n:127.0.0.1:64102_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 987132 INFO (qtp1217080757-7074) [n:127.0.0.1:64103_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 987134 INFO (qtp1913120776-7085) [n:127.0.0.1:64102_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica1] o.a.s.c.CoreContainer Creating SolrCore 'addReplicaColl_shard2_replica1' using configuration from collection addReplicaColl
[junit4] 2> 987135 INFO (qtp1217080757-7074) [n:127.0.0.1:64103_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica2] o.a.s.c.CoreContainer Creating SolrCore 'addReplicaColl_shard2_replica2' using configuration from collection addReplicaColl
[junit4] 2> 987135 INFO (qtp1913120776-7085) [n:127.0.0.1:64102_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica1] o.a.s.c.SolrCore [[addReplicaColl_shard2_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-001/node4/addReplicaColl_shard2_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-001/node4/./addReplicaColl_shard2_replica1/data/]
[junit4] 2> 987135 INFO (qtp1217080757-7074) [n:127.0.0.1:64103_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica2] o.a.s.c.SolrCore [[addReplicaColl_shard2_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-001/node3/addReplicaColl_shard2_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-001/node3/./addReplicaColl_shard2_replica2/data/]
[junit4] 2> 987139 INFO (qtp1035706113-7067) [n:127.0.0.1:64100_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica1] o.a.s.s.IndexSchema [addReplicaColl_shard1_replica1] Schema name=minimal
[junit4] 2> 987141 INFO (qtp1035706113-7067) [n:127.0.0.1:64100_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 987142 INFO (qtp1035706113-7067) [n:127.0.0.1:64100_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'addReplicaColl_shard1_replica1' using configuration from collection addReplicaColl
[junit4] 2> 987142 INFO (qtp1035706113-7067) [n:127.0.0.1:64100_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica1] o.a.s.c.SolrCore [[addReplicaColl_shard1_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-001/node1/addReplicaColl_shard1_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_C8AE32E10101EF03-001/tempDir-001/node1/./addReplicaColl_shard1_replica1/data/]
[junit4] 2> 987165 INFO (qtp2076956430-7080) [n:127.0.0.1:64101_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 987165 INFO (qtp2076956430-7080) [n:127.0.0.1:64101_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 987167 INFO (qtp1913120776-7085) [n:127.0.0.1:64102_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 987167 INFO (qtp1913120776-7085) [n:127.0.0.1:64102_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 987168 INFO (qtp1913120776-7085) [n:127.0.0.1:64102_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 987168 INFO (qtp1913120776-7085) [n:127.0.0.1:64102_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 987170 INFO (qtp2076956430-7080) [n:127.0.0.1:64101_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 987170 INFO (qtp2076956430-7080) [n:127.0.0.1:64101_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 987170 INFO (qtp1035706113-7067) [n:127.0.0.1:64100_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 987170 INFO (qtp1035706113-7067) [n:127.0.0.1:64100_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 987186 INFO (qtp1035706113-7067) [n:127.0.0.1:64100_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 987186 INFO (qtp1035706113-7067) [n:127.0.0.1:64100_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 987187 INFO (qtp2076956430-7080) [n:127.0.0.1:64101_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@31690811[addReplicaColl_shard1_replica2] main]
[junit4] 2> 987189 INFO (qtp2076956430-7080) [n:127.0.0.1:64101_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 987190 INFO (qtp2076956430-7080) [n:127.0.0.1:64101_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 987190 INFO (qtp2076956430-7080) [n:127.0.0.1:64101_solr c:addReplicaColl s:shard1 x
[...truncated too long message...]
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:357)
[junit4] 2> at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:263)
[junit4] 2> at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:397)
[junit4] 2> at org.apache.solr.handler.ReplicationHandler.lambda$handleRequestBody$0(ReplicationHandler.java:279)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> Caused by: org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://127.0.0.1:50716/solr/collection1
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:634)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:279)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268)
[junit4] 2> at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
[junit4] 2> at org.apache.solr.handler.IndexFetcher.fetchFileList(IndexFetcher.java:239)
[junit4] 2> ... 5 more
[junit4] 2> Caused by: org.apache.http.conn.HttpHostConnectException: Connect to 127.0.0.1:50716 [/127.0.0.1] failed: Connection refused (Connection refused)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:151)
[junit4] 2> at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
[junit4] 2> at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
[junit4] 2> at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
[junit4] 2> at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
[junit4] 2> at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
[junit4] 2> at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
[junit4] 2> at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:532)
[junit4] 2> ... 9 more
[junit4] 2> Caused by: java.net.ConnectException: Connection refused (Connection refused)
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:74)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
[junit4] 2> ... 19 more
[junit4] 2>
[junit4] 2> 2195673 INFO (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[C8AE32E10101EF03]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 2195673 INFO (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[C8AE32E10101EF03]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@72d4bce1{/solr,null,UNAVAILABLE}
[junit4] 2> 2195675 INFO (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[C8AE32E10101EF03]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@2fa4989f{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 2195675 INFO (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[C8AE32E10101EF03]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1287579614
[junit4] 2> 2195676 INFO (coreCloseExecutor-7811-thread-1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@6cfc35b6
[junit4] 2> 2195677 INFO (coreCloseExecutor-7811-thread-1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
[junit4] 2> 2195677 INFO (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[C8AE32E10101EF03]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 2195678 INFO (TEST-TestReplicationHandler.doTestReplicateAfterWrite2Slave-seed#[C8AE32E10101EF03]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3ec23423{/solr,null,UNAVAILABLE}
[junit4] 2> 2195680 INFO (SUITE-TestReplicationHandler-seed#[C8AE32E10101EF03]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_C8AE32E10101EF03-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=true): {}, locale=he-IL, timezone=Europe/Istanbul
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=90226336,total=305999872
[junit4] 2> NOTE: All tests run in this JVM: [OverseerTaskQueueTest, TestJmxMonitoredMap, TestRandomRequestDistribution, ReplaceNodeTest, DistributedIntervalFacetingTest, TestUtils, TestSchemaManager, DistributedFacetPivotLongTailTest, TestSolrIndexConfig, CollectionsAPISolrJTest, OpenCloseCoreStressTest, TestOmitPositions, TestBulkSchemaAPI, TestReqParamsAPI, LeaderFailureAfterFreshStartTest, BlockCacheTest, SolrSlf4jReporterTest, MinimalSchemaTest, TestSchemaNameResource, DistributedTermsComponentTest, TestManagedSchema, TestSlowCompositeReaderWrapper, ExternalFileFieldSortTest, TestRequestStatusCollectionAPI, IgnoreCommitOptimizeUpdateProcessorFactoryTest, CursorMarkTest, TestSolrCoreSnapshots, TestUseDocValuesAsStored, TestAddFieldRealTimeGet, IndexSchemaTest, UpdateRequestProcessorFactoryTest, DebugComponentTest, TemplateUpdateProcessorTest, MetricsHandlerTest, URLClassifyProcessorTest, HighlighterMaxOffsetTest, TestExpandComponent, FileBasedSpellCheckerTest, QueryParsingTest, TestLeaderElectionWithEmptyReplica, ForceLeaderTest, PreAnalyzedFieldTest, TestCollapseQParserPlugin, TestReplicaProperties, TestLuceneMatchVersion, ResourceLoaderTest, BJQParserTest, CdcrBootstrapTest, BasicDistributedZk2Test, SegmentsInfoRequestHandlerTest, ClusterStateTest, TestFoldingMultitermQuery, OutOfBoxZkACLAndCredentialsProvidersTest, TestCloudPivotFacet, BadIndexSchemaTest, TestSolrDeletionPolicy2, TestFieldCacheSanityChecker, TestManagedStopFilterFactory, TlogReplayBufferedWhileIndexingTest, SolrMetricManagerTest, TestWriterPerf, TestQueryTypes, FullHLLTest, TestAtomicUpdateErrorCases, DistanceUnitsTest, TestBlobHandler, TestCollectionAPI, DirectoryFactoryTest, TestSolrConfigHandler, ConnectionReuseTest, TestFuzzyAnalyzedSuggestions, DeleteNodeTest, TestFiltering, PreAnalyzedUpdateProcessorTest, TestNonDefinedSimilarityFactory, CacheHeaderTest, SuggesterTest, TestSegmentSorting, MetricUtilsTest, SuggesterTSTTest, RuleEngineTest, SliceStateTest, ReplicaListTransformerTest, ManagedSchemaRoundRobinCloudTest, RestartWhileUpdatingTest, CoreAdminCreateDiscoverTest, FieldMutatingUpdateProcessorTest, DistributedDebugComponentTest, TestValueSourceCache, BitVectorTest, SpellCheckCollatorTest, TestCloudRecovery, BigEndianAscendingWordDeserializerTest, BufferStoreTest, HighlighterConfigTest, JvmMetricsTest, TestSubQueryTransformerDistrib, TestFieldCacheWithThreads, TestComponentsName, DefaultValueUpdateProcessorTest, TestInitQParser, CollectionsAPIDistributedZkTest, TestDistributedMissingSort, HLLUtilTest, TestClassicSimilarityFactory, TestReload, TestIntervalFaceting, TermVectorComponentDistributedTest, TestOnReconnectListenerSupport, HdfsBasicDistributedZkTest, TestDynamicFieldCollectionResource, TestCoreDiscovery, TestMaxScoreQueryParser, BlobRepositoryCloudTest, BlockJoinFacetRandomTest, SolrGraphiteReporterTest, SortByFunctionTest, TestFieldCollectionResource, TestExactStatsCache, TestSolrQueryParser, TestSolr4Spatial, TestCustomDocTransformer, BasicFunctionalityTest, AutoCommitTest, TestGeoJSONResponseWriter, TestJsonFacets, CurrencyFieldOpenExchangeTest, MissingSegmentRecoveryTest, TestExceedMaxTermLength, FieldAnalysisRequestHandlerTest, TestFieldTypeCollectionResource, SolrCmdDistributorTest, HttpSolrCallGetCoreTest, SolrCoreCheckLockOnStartupTest, CloneFieldUpdateProcessorFactoryTest, ChangedSchemaMergeTest, TestJoin, TestEmbeddedSolrServerConstructors, TestBadConfig, TestUninvertingReader, TestFieldCache, RecoveryAfterSoftCommitTest, TestScoreJoinQPScore, TestConfigSets, TestLMDirichletSimilarityFactory, LeaderElectionIntegrationTest, FullSolrCloudDistribCmdsTest, TestSchemaSimilarityResource, TestTolerantUpdateProcessorCloud, RAMDirectoryFactoryTest, ShardSplitTest, TestDistributedGrouping, ShardRoutingCustomTest, TestConfigSetImmutable, TestStressCloudBlindAtomicUpdates, JsonLoaderTest, TestImplicitCoreProperties, TestQuerySenderListener, CollectionTooManyReplicasTest, RequestHandlersTest, ShowFileRequestHandlerTest, DirectUpdateHandlerOptimizeTest, HdfsDirectoryFactoryTest, TestGraphTermsQParserPlugin, SSLMigrationTest, TestImpersonationWithHadoopAuth, NotRequiredUniqueKeyTest, ChaosMonkeySafeLeaderTest, OverseerTest, ShardRoutingTest, BasicZkTest, TestReplicationHandler]
[junit4] Completed [347/680 (3!)] on J1 in 103.23s, 15 tests, 1 failure, 1 skipped <<< FAILURES!
[...truncated 63548 lines...]
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3801
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3801/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC
1 tests failed.
FAILED: org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit
Error Message:
expected:<1> but was:<2>
Stack Trace:
java.lang.AssertionError: expected:<1> but was:<2>
at __randomizedtesting.SeedInfo.seed([EE51548B2A6144F4:171CC7241614097E]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:283)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 10962 lines...]
[junit4] Suite: org.apache.solr.cloud.ShardSplitTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/init-core-data-001
[junit4] 2> 240058 INFO (SUITE-ShardSplitTest-seed#[EE51548B2A6144F4]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields
[junit4] 2> 240062 INFO (SUITE-ShardSplitTest-seed#[EE51548B2A6144F4]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 240062 INFO (SUITE-ShardSplitTest-seed#[EE51548B2A6144F4]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /pqmsy/
[junit4] 2> 240064 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 240065 INFO (Thread-384) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 240065 INFO (Thread-384) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 240170 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ZkTestServer start zk server on port:52694
[junit4] 2> 240205 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 240209 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 240213 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 240216 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 240219 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 240222 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 240225 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 240228 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 240232 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 240235 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 240238 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 240542 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/control-001/cores/collection1
[junit4] 2> 240544 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 240546 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@635d75fc{/pqmsy,null,AVAILABLE}
[junit4] 2> 240548 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3036c676{HTTP/1.1,[http/1.1]}{127.0.0.1:52697}
[junit4] 2> 240548 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server Started @248937ms
[junit4] 2> 240548 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/tempDir-001/control/data, hostContext=/pqmsy, hostPort=52697, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/control-001/cores}
[junit4] 2> 240548 ERROR (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 240548 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 240548 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 240548 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 240549 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-27T04:00:08.366Z
[junit4] 2> 240554 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 240554 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/control-001/solr.xml
[junit4] 2> 240566 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52694/solr
[junit4] 2> 240623 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:52697_pqmsy
[junit4] 2> 240626 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.Overseer Overseer (id=97353046942285828-127.0.0.1:52697_pqmsy-n_0000000000) starting
[junit4] 2> 240647 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52697_pqmsy
[junit4] 2> 240652 INFO (OverseerStateUpdate-97353046942285828-127.0.0.1:52697_pqmsy-n_0000000000) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 240750 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/control-001/cores
[junit4] 2> 240750 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 240756 INFO (OverseerStateUpdate-97353046942285828-127.0.0.1:52697_pqmsy-n_0000000000) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 241788 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 241803 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 241927 WARN (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 241928 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 241971 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 241972 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/control-001/cores/collection1/data/]
[junit4] 2> 241972 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@da160ff
[junit4] 2> 241976 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=5.63671875, floorSegmentMB=0.2470703125, forceMergeDeletesPctAllowed=22.017257614259755, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8726594395281767
[junit4] 2> 241987 WARN (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 242006 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 242006 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 242008 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 242008 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 242009 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 242010 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@58833da0[collection1] main]
[junit4] 2> 242013 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 242014 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 242014 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 242016 INFO (searcherExecutor-935-thread-1-processing-n:127.0.0.1:52697_pqmsy x:collection1 c:control_collection) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@58833da0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 242017 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557648753121296384
[junit4] 2> 242034 INFO (coreZkRegister-927-thread-1-processing-n:127.0.0.1:52697_pqmsy x:collection1 c:control_collection) [n:127.0.0.1:52697_pqmsy c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 242034 INFO (coreZkRegister-927-thread-1-processing-n:127.0.0.1:52697_pqmsy x:collection1 c:control_collection) [n:127.0.0.1:52697_pqmsy c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 242034 INFO (coreZkRegister-927-thread-1-processing-n:127.0.0.1:52697_pqmsy x:collection1 c:control_collection) [n:127.0.0.1:52697_pqmsy c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52697/pqmsy/collection1/
[junit4] 2> 242034 INFO (coreZkRegister-927-thread-1-processing-n:127.0.0.1:52697_pqmsy x:collection1 c:control_collection) [n:127.0.0.1:52697_pqmsy c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 242034 INFO (coreZkRegister-927-thread-1-processing-n:127.0.0.1:52697_pqmsy x:collection1 c:control_collection) [n:127.0.0.1:52697_pqmsy c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:52697/pqmsy/collection1/ has no replicas
[junit4] 2> 242044 INFO (coreZkRegister-927-thread-1-processing-n:127.0.0.1:52697_pqmsy x:collection1 c:control_collection) [n:127.0.0.1:52697_pqmsy c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52697/pqmsy/collection1/ shard1
[junit4] 2> 242159 INFO (coreZkRegister-927-thread-1-processing-n:127.0.0.1:52697_pqmsy x:collection1 c:control_collection) [n:127.0.0.1:52697_pqmsy c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 242407 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 242410 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52694/solr ready
[junit4] 2> 242410 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 242410 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
[junit4] 2> 242856 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-1-001/cores/collection1
[junit4] 2> 242858 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-1-001
[junit4] 2> 242859 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 242887 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7ff538ec{/pqmsy,null,AVAILABLE}
[junit4] 2> 242887 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2fd4f8ca{HTTP/1.1,[http/1.1]}{127.0.0.1:52707}
[junit4] 2> 242888 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server Started @251276ms
[junit4] 2> 242888 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/pqmsy, hostPort=52707, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-1-001/cores}
[junit4] 2> 242888 ERROR (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 242889 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 242889 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 242889 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 242889 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-27T04:00:10.706Z
[junit4] 2> 242897 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 242897 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-1-001/solr.xml
[junit4] 2> 242915 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52694/solr
[junit4] 2> 242940 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 242954 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52707_pqmsy
[junit4] 2> 242958 INFO (zkCallback-364-thread-2-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 242959 INFO (zkCallback-368-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 242959 INFO (zkCallback-374-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 243093 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-1-001/cores
[junit4] 2> 243093 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 243099 INFO (OverseerStateUpdate-97353046942285828-127.0.0.1:52697_pqmsy-n_0000000000) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
[junit4] 2> 243213 INFO (zkCallback-374-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 244125 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 244157 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 244218 WARN (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 244220 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 244259 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 244259 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 244260 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@da160ff
[junit4] 2> 244262 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=5.63671875, floorSegmentMB=0.2470703125, forceMergeDeletesPctAllowed=22.017257614259755, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8726594395281767
[junit4] 2> 244285 WARN (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 244303 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 244304 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 244304 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 244304 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 244305 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 244306 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@384dcbe1[collection1] main]
[junit4] 2> 244309 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 244310 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 244310 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 244311 INFO (searcherExecutor-946-thread-1-processing-n:127.0.0.1:52707_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@384dcbe1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 244312 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557648755527778304
[junit4] 2> 244325 INFO (coreZkRegister-940-thread-1-processing-n:127.0.0.1:52707_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 244325 INFO (coreZkRegister-940-thread-1-processing-n:127.0.0.1:52707_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 244325 INFO (coreZkRegister-940-thread-1-processing-n:127.0.0.1:52707_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52707/pqmsy/collection1/
[junit4] 2> 244325 INFO (coreZkRegister-940-thread-1-processing-n:127.0.0.1:52707_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 244325 INFO (coreZkRegister-940-thread-1-processing-n:127.0.0.1:52707_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:52707/pqmsy/collection1/ has no replicas
[junit4] 2> 244335 INFO (coreZkRegister-940-thread-1-processing-n:127.0.0.1:52707_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52707/pqmsy/collection1/ shard2
[junit4] 2> 244451 INFO (zkCallback-374-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 244505 INFO (coreZkRegister-940-thread-1-processing-n:127.0.0.1:52707_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 244625 INFO (zkCallback-374-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 245107 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-2-001/cores/collection1
[junit4] 2> 245110 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-2-001
[junit4] 2> 245114 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 245117 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@52ea8fd8{/pqmsy,null,AVAILABLE}
[junit4] 2> 245118 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@afccc17{HTTP/1.1,[http/1.1]}{127.0.0.1:52715}
[junit4] 2> 245118 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server Started @253507ms
[junit4] 2> 245118 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/pqmsy, hostPort=52715, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-2-001/cores}
[junit4] 2> 245118 ERROR (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 245124 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 245124 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 245125 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 245125 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-27T04:00:12.942Z
[junit4] 2> 245130 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 245130 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-2-001/solr.xml
[junit4] 2> 245148 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52694/solr
[junit4] 2> 245169 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 245179 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52715_pqmsy
[junit4] 2> 245185 INFO (zkCallback-368-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 245187 INFO (zkCallback-380-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 245186 INFO (zkCallback-364-thread-3-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 245185 INFO (zkCallback-374-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 245261 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-2-001/cores
[junit4] 2> 245261 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 245265 INFO (OverseerStateUpdate-97353046942285828-127.0.0.1:52697_pqmsy-n_0000000000) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 245378 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 245378 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 246284 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 246325 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 246409 WARN (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 246411 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 246453 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 246453 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 246454 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@da160ff
[junit4] 2> 246459 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=5.63671875, floorSegmentMB=0.2470703125, forceMergeDeletesPctAllowed=22.017257614259755, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8726594395281767
[junit4] 2> 246474 WARN (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 246499 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 246499 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 246501 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 246501 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 246502 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 246504 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4970dade[collection1] main]
[junit4] 2> 246506 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 246507 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 246507 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 246509 INFO (searcherExecutor-957-thread-1-processing-n:127.0.0.1:52715_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4970dade[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 246510 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557648757832548352
[junit4] 2> 246524 INFO (coreZkRegister-951-thread-1-processing-n:127.0.0.1:52715_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52715_pqmsy c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 246524 INFO (coreZkRegister-951-thread-1-processing-n:127.0.0.1:52715_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52715_pqmsy c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 246524 INFO (coreZkRegister-951-thread-1-processing-n:127.0.0.1:52715_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52715_pqmsy c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52715/pqmsy/collection1/
[junit4] 2> 246524 INFO (coreZkRegister-951-thread-1-processing-n:127.0.0.1:52715_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52715_pqmsy c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 246524 INFO (coreZkRegister-951-thread-1-processing-n:127.0.0.1:52715_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52715_pqmsy c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:52715/pqmsy/collection1/ has no replicas
[junit4] 2> 246535 INFO (coreZkRegister-951-thread-1-processing-n:127.0.0.1:52715_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52715_pqmsy c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52715/pqmsy/collection1/ shard1
[junit4] 2> 246645 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 246645 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 246697 INFO (coreZkRegister-951-thread-1-processing-n:127.0.0.1:52715_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52715_pqmsy c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 246807 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 246807 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 247174 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-3-001/cores/collection1
[junit4] 2> 247177 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-3-001
[junit4] 2> 247177 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 247181 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@387f9911{/pqmsy,null,AVAILABLE}
[junit4] 2> 247182 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@43ac5d2e{HTTP/1.1,[http/1.1]}{127.0.0.1:52724}
[junit4] 2> 247182 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server Started @255570ms
[junit4] 2> 247182 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/pqmsy, hostPort=52724, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-3-001/cores}
[junit4] 2> 247182 ERROR (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 247183 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 247183 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 247184 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 247184 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-27T04:00:15.001Z
[junit4] 2> 247193 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 247193 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-3-001/solr.xml
[junit4] 2> 247225 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52694/solr
[junit4] 2> 247250 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 247262 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52724_pqmsy
[junit4] 2> 247266 INFO (zkCallback-364-thread-3-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 247267 INFO (zkCallback-386-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 247267 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 247266 INFO (zkCallback-368-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 247266 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 247307 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-3-001/cores
[junit4] 2> 247307 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 247322 INFO (OverseerStateUpdate-97353046942285828-127.0.0.1:52697_pqmsy-n_0000000000) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
[junit4] 2> 247433 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 247433 INFO (zkCallback-386-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 247433 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 248344 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 248357 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 248450 WARN (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 248452 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 248490 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 248490 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 248490 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@da160ff
[junit4] 2> 248493 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=5.63671875, floorSegmentMB=0.2470703125, forceMergeDeletesPctAllowed=22.017257614259755, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8726594395281767
[junit4] 2> 248510 WARN (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 248531 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 248531 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 248532 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 248532 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 248534 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 248534 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@56dcf754[collection1] main]
[junit4] 2> 248536 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 248537 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 248537 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 248539 INFO (searcherExecutor-968-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@56dcf754[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 248540 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557648759961157632
[junit4] 2> 248547 INFO (coreZkRegister-962-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 248547 INFO (updateExecutor-383-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 248548 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 248548 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 248548 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 248548 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 248548 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:52707/pqmsy/collection1/] and I am [http://127.0.0.1:52724/pqmsy/collection1/]
[junit4] 2> 248552 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:52707/pqmsy]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:52724_pqmsy&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 248556 INFO (qtp407216533-1916) [n:127.0.0.1:52707_pqmsy ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 248557 INFO (qtp407216533-1916) [n:127.0.0.1:52707_pqmsy ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard2 of collection1) have state: recovering
[junit4] 2> 248557 INFO (qtp407216533-1916) [n:127.0.0.1:52707_pqmsy ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:52724_pqmsy, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:52724/pqmsy","node_name":"127.0.0.1:52724_pqmsy","state":"down"}
[junit4] 2> 248669 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 248669 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 248669 INFO (zkCallback-386-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 249198 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-4-001/cores/collection1
[junit4] 2> 249199 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-4-001
[junit4] 2> 249199 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 249201 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@e8f82a8{/pqmsy,null,AVAILABLE}
[junit4] 2> 249202 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@10b00fc1{HTTP/1.1,[http/1.1]}{127.0.0.1:52733}
[junit4] 2> 249202 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server Started @257590ms
[junit4] 2> 249202 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/tempDir-001/jetty4, solrconfig=solrconfig.xml, hostContext=/pqmsy, hostPort=52733, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-4-001/cores}
[junit4] 2> 249203 ERROR (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 249203 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 249203 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 249203 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 249203 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-27T04:00:17.020Z
[junit4] 2> 249209 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 249209 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-4-001/solr.xml
[junit4] 2> 249237 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52694/solr
[junit4] 2> 249259 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52733_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 249270 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52733_pqmsy ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52733_pqmsy
[junit4] 2> 249277 INFO (zkCallback-386-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 249285 INFO (zkCallback-393-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 249284 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 249277 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 249277 INFO (zkCallback-364-thread-3-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 249277 INFO (zkCallback-368-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 249317 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52733_pqmsy ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-4-001/cores
[junit4] 2> 249317 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52733_pqmsy ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 249324 INFO (OverseerStateUpdate-97353046942285828-127.0.0.1:52697_pqmsy-n_0000000000) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 249433 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 249433 INFO (zkCallback-393-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 249433 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 249433 INFO (zkCallback-386-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 249567 INFO (qtp407216533-1916) [n:127.0.0.1:52707_pqmsy ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:52724_pqmsy, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:52724/pqmsy","node_name":"127.0.0.1:52724_pqmsy","state":"recovering"}
[junit4] 2> 249567 INFO (qtp407216533-1916) [n:127.0.0.1:52707_pqmsy ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 249568 INFO (qtp407216533-1916) [n:127.0.0.1:52707_pqmsy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:52724_pqmsy&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1011
[junit4] 2> 250346 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 250359 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 250430 WARN (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 250432 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 250473 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 250473 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-4-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-4-001/cores/collection1/data/]
[junit4] 2> 250473 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@da160ff
[junit4] 2> 250476 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=5.63671875, floorSegmentMB=0.2470703125, forceMergeDeletesPctAllowed=22.017257614259755, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8726594395281767
[junit4] 2> 250484 WARN (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 250502 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 250502 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 250503 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 250503 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 250504 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 250505 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@13e03e7[collection1] main]
[junit4] 2> 250507 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 250507 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 250508 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 250509 INFO (searcherExecutor-979-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@13e03e7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 250509 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557648762025803776
[junit4] 2> 250516 INFO (coreZkRegister-973-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52733_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 250516 INFO (updateExecutor-390-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:52733_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 250517 INFO (recoveryExecutor-391-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:52733_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 250517 INFO (recoveryExecutor-391-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:52733_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 250517 INFO (recoveryExecutor-391-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:52733_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 250517 INFO (recoveryExecutor-391-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:52733_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 250517 INFO (recoveryExecutor-391-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:52733_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:52715/pqmsy/collection1/] and I am [http://127.0.0.1:52733/pqmsy/collection1/]
[junit4] 2> 250520 INFO (recoveryExecutor-391-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:52733_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:52715/pqmsy]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:52733_pqmsy&coreNodeName=core_node4&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 250524 INFO (qtp1031597028-1944) [n:127.0.0.1:52715_pqmsy ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node4, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 250526 INFO (qtp1031597028-1944) [n:127.0.0.1:52715_pqmsy ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 250526 INFO (qtp1031597028-1944) [n:127.0.0.1:52715_pqmsy ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:52733_pqmsy, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{"core":"collection1","base_url":"http://127.0.0.1:52733/pqmsy","node_name":"127.0.0.1:52733_pqmsy","state":"down"}
[junit4] 2> 250638 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 250638 INFO (zkCallback-393-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 250638 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 250638 INFO (zkCallback-386-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 250928 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 250928 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 15 for each attempt
[junit4] 2> 250928 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):15
[junit4] 2> 251527 INFO (qtp1031597028-1944) [n:127.0.0.1:52715_pqmsy ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:52733_pqmsy, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{"core":"collection1","base_url":"http://127.0.0.1:52733/pqmsy","node_name":"127.0.0.1:52733_pqmsy","state":"recovering"}
[junit4] 2> 251527 INFO (qtp1031597028-1944) [n:127.0.0.1:52715_pqmsy ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node4, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 251527 INFO (qtp1031597028-1944) [n:127.0.0.1:52715_pqmsy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:52733_pqmsy&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node4&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1003
[junit4] 2> 256569 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:52707/pqmsy/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 256569 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:52724/pqmsy START replicas=[http://127.0.0.1:52707/pqmsy/collection1/] nUpdates=100
[junit4] 2> 256575 INFO (qtp407216533-1920) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 256575 INFO (qtp407216533-1920) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp=/pqmsy path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 256577 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 256577 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 256577 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1
[...truncated too long message...]
:testSplitStaticIndexReplication_shard1_0_replica1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.testSplitStaticIndexReplication.shard1_0.replica1
[junit4] 2> 539330 INFO (recoveryExecutor-581-thread-1-processing-n:127.0.0.1:53301_pqmsy x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53301_pqmsy c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
[junit4] 2> 539333 WARN (zkCallback-583-thread-1-processing-n:127.0.0.1:53301_pqmsy) [n:127.0.0.1:53301_pqmsy ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 539334 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 539334 INFO (zkCallback-549-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 539334 INFO (zkCallback-573-thread-1-processing-n:127.0.0.1:53314_pqmsy) [n:127.0.0.1:53314_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 539335 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@2d1c06b6{HTTP/1.1,[http/1.1]}{127.0.0.1:53301}
[junit4] 2> 539335 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1a5d0cca{/pqmsy,null,UNAVAILABLE}
[junit4] 2> 539337 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 53306
[junit4] 2> 539337 INFO (zkCallback-566-thread-2-processing-n:127.0.0.1:53306_pqmsy) [n:127.0.0.1:53306_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 539337 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=879483284
[junit4] 2> 539342 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.Overseer Overseer (id=97353063404273678-127.0.0.1:53306_pqmsy-n_0000000003) closing
[junit4] 2> 539342 INFO (OverseerStateUpdate-97353063404273678-127.0.0.1:53306_pqmsy-n_0000000003) [n:127.0.0.1:53306_pqmsy ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:53306_pqmsy
[junit4] 2> 539346 INFO (zkCallback-549-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 539346 INFO (zkCallback-573-thread-1-processing-n:127.0.0.1:53314_pqmsy) [n:127.0.0.1:53314_pqmsy ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:53314_pqmsy
[junit4] 2> 539346 INFO (zkCallback-573-thread-2-processing-n:127.0.0.1:53314_pqmsy) [n:127.0.0.1:53314_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 539349 INFO (zkCallback-573-thread-1-processing-n:127.0.0.1:53314_pqmsy) [n:127.0.0.1:53314_pqmsy ] o.a.s.c.Overseer Overseer (id=97353063404273681-127.0.0.1:53314_pqmsy-n_0000000004) starting
[junit4] 2> 540856 WARN (zkCallback-566-thread-3-processing-n:127.0.0.1:53306_pqmsy) [n:127.0.0.1:53306_pqmsy ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 541573 WARN (zkCallback-566-thread-1-processing-n:127.0.0.1:53306_pqmsy) [n:127.0.0.1:53306_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.SyncStrategy Closed, skipping sync up.
[junit4] 2> 541574 INFO (zkCallback-566-thread-1-processing-n:127.0.0.1:53306_pqmsy) [n:127.0.0.1:53306_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 541574 INFO (zkCallback-566-thread-1-processing-n:127.0.0.1:53306_pqmsy) [n:127.0.0.1:53306_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@5a8cf277
[junit4] 2> 541624 INFO (zkCallback-566-thread-1-processing-n:127.0.0.1:53306_pqmsy) [n:127.0.0.1:53306_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
[junit4] 2> 541625 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 541627 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@3e90c0e{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 541627 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@51894b1{/pqmsy,null,UNAVAILABLE}
[junit4] 2> 541628 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 53314
[junit4] 2> 541628 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=509049753
[junit4] 2> 541632 INFO (coreCloseExecutor-1596-thread-1) [n:127.0.0.1:53314_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@4379f09f
[junit4] 2> 541656 INFO (coreCloseExecutor-1596-thread-1) [n:127.0.0.1:53314_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
[junit4] 2> 541658 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.Overseer Overseer (id=97353063404273681-127.0.0.1:53314_pqmsy-n_0000000004) closing
[junit4] 2> 541658 INFO (OverseerStateUpdate-97353063404273681-127.0.0.1:53314_pqmsy-n_0000000004) [n:127.0.0.1:53314_pqmsy ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:53314_pqmsy
[junit4] 2> 541663 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 541664 INFO (zkCallback-549-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 541664 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@693bff5a{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 541665 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6a67600c{/pqmsy,null,UNAVAILABLE}
[junit4] 2> 541669 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 53282
[junit4] 2> 541670 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53279 53279
[junit4] 2> 541685 INFO (Thread-638) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53279 53279
[junit4] 2> 543753 WARN (Thread-638) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/aliases.json
[junit4] 2> 8 /solr/clusterprops.json
[junit4] 2> 6 /solr/security.json
[junit4] 2> 6 /solr/configs/conf1
[junit4] 2> 5 /solr/collections/collection1/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/clusterstate.json
[junit4] 2> 4 /solr/collections/testSplitStaticIndexReplication/state.json
[junit4] 2> 2 /solr/overseer_elect/election/97353063404273672-127.0.0.1:53295_pqmsy-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 87 /solr/overseer/queue
[junit4] 2> 38 /solr/overseer/collection-queue-work
[junit4] 2> 20 /solr/overseer/queue-work
[junit4] 2> 8 /solr/collections
[junit4] 2> 7 /solr/live_nodes
[junit4] 2>
[junit4] 2> 543754 INFO (SUITE-ShardSplitTest-seed#[EE51548B2A6144F4]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=false): {}, locale=sr-ME, timezone=America/Fort_Nelson
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=265749584,total=471130112
[junit4] 2> NOTE: All tests run in this JVM: [FieldMutatingUpdateProcessorTest, DistributedTermsComponentTest, TestOverriddenPrefixQueryForCustomFieldType, CSVRequestHandlerTest, RAMDirectoryFactoryTest, AutoCommitTest, TestLRUStatsCache, TestNRTOpen, TestAnalyzedSuggestions, TestSimpleQParserPlugin, TestDynamicLoading, TestMinMaxOnMultiValuedField, BlockDirectoryTest, HighlighterConfigTest, BlockJoinFacetSimpleTest, TestPHPSerializedResponseWriter, LoggingHandlerTest, FullSolrCloudDistribCmdsTest, SliceStateTest, ShowFileRequestHandlerTest, AliasIntegrationTest, SolrInfoMBeanTest, ShardSplitTest]
[junit4] Completed [89/684 (1!)] on J1 in 303.72s, 5 tests, 1 failure <<< FAILURES!
[...truncated 64363 lines...]
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3800
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3800/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
1 tests failed.
FAILED: org.apache.solr.cloud.PeerSyncReplicationTest.test
Error Message:
timeout waiting to see all nodes active
Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
at __randomizedtesting.SeedInfo.seed([A0E5E2CB84D8E147:28B1DD112A248CBF]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12286 lines...]
[junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/init-core-data-001
[junit4] 2> 2641237 INFO (SUITE-PeerSyncReplicationTest-seed#[A0E5E2CB84D8E147]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields
[junit4] 2> 2641241 INFO (SUITE-PeerSyncReplicationTest-seed#[A0E5E2CB84D8E147]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 2641242 INFO (SUITE-PeerSyncReplicationTest-seed#[A0E5E2CB84D8E147]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 2641244 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2641244 INFO (Thread-2880) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2641244 INFO (Thread-2880) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2641347 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ZkTestServer start zk server on port:52443
[junit4] 2> 2641380 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 2641448 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 2641561 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 2641650 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 2641687 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 2641712 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 2641744 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 2641794 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 2641813 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 2641830 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 2641855 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 2645831 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/control-001/cores/collection1
[junit4] 2> 2645859 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2645864 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7a9f2e46{/,null,AVAILABLE}
[junit4] 2> 2645879 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@8cd1f39{SSL,[ssl, http/1.1]}{127.0.0.1:52473}
[junit4] 2> 2645880 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.Server Started @2651354ms
[junit4] 2> 2645880 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/tempDir-001/control/data, hostContext=/, hostPort=52473, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/control-001/cores}
[junit4] 2> 2645880 ERROR (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2645881 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2645881 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2645881 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2645881 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-26T19:57:03.824Z
[junit4] 2> 2645888 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2645888 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/control-001/solr.xml
[junit4] 2> 2645905 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52443/solr
[junit4] 2> 2645969 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52473_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:52473_
[junit4] 2> 2645971 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52473_ ] o.a.s.c.Overseer Overseer (id=97351147139235845-127.0.0.1:52473_-n_0000000000) starting
[junit4] 2> 2645990 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52473_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52473_
[junit4] 2> 2645995 INFO (zkCallback-2962-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2646098 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52473_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/control-001/cores
[junit4] 2> 2646099 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52473_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2646108 INFO (OverseerStateUpdate-97351147139235845-127.0.0.1:52473_-n_0000000000) [n:127.0.0.1:52473_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2647138 WARN (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 2647139 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2647169 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2647306 WARN (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2647308 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2647340 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 2647340 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/control-001/cores/collection1/data/]
[junit4] 2> 2647341 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d92275b
[junit4] 2> 2647346 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1672653504, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2647368 WARN (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2647388 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2647388 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2647390 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2647390 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2647390 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=55.8642578125, floorSegmentMB=0.7197265625, forceMergeDeletesPctAllowed=2.102139642005424, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2647391 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@692f0eb4[collection1] main]
[junit4] 2> 2647393 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2647394 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2647394 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2647396 INFO (searcherExecutor-6484-thread-1-processing-n:127.0.0.1:52473_ x:collection1 c:control_collection) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@692f0eb4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2647396 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557618360675467264
[junit4] 2> 2647411 INFO (coreZkRegister-6476-thread-1-processing-n:127.0.0.1:52473_ x:collection1 c:control_collection) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2647411 INFO (coreZkRegister-6476-thread-1-processing-n:127.0.0.1:52473_ x:collection1 c:control_collection) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2647411 INFO (coreZkRegister-6476-thread-1-processing-n:127.0.0.1:52473_ x:collection1 c:control_collection) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:52473/collection1/
[junit4] 2> 2647412 INFO (coreZkRegister-6476-thread-1-processing-n:127.0.0.1:52473_ x:collection1 c:control_collection) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2647412 INFO (coreZkRegister-6476-thread-1-processing-n:127.0.0.1:52473_ x:collection1 c:control_collection) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy https://127.0.0.1:52473/collection1/ has no replicas
[junit4] 2> 2647422 INFO (coreZkRegister-6476-thread-1-processing-n:127.0.0.1:52473_ x:collection1 c:control_collection) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:52473/collection1/ shard1
[junit4] 2> 2647595 INFO (coreZkRegister-6476-thread-1-processing-n:127.0.0.1:52473_ x:collection1 c:control_collection) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2647714 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2647717 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52443/solr ready
[junit4] 2> 2647717 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 2648135 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-1-001/cores/collection1
[junit4] 2> 2648137 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-1-001
[junit4] 2> 2648142 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2648150 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2c2437e0{/,null,AVAILABLE}
[junit4] 2> 2648152 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4f40effb{SSL,[ssl, http/1.1]}{127.0.0.1:52478}
[junit4] 2> 2648152 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.Server Started @2653627ms
[junit4] 2> 2648152 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=52478, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-1-001/cores}
[junit4] 2> 2648152 ERROR (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2648153 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2648153 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2648153 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2648153 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-26T19:57:06.096Z
[junit4] 2> 2648158 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2648158 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-1-001/solr.xml
[junit4] 2> 2648173 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52443/solr
[junit4] 2> 2648195 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52478_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2648219 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52478_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52478_
[junit4] 2> 2648224 INFO (zkCallback-2962-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2648224 INFO (zkCallback-2966-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2648225 INFO (zkCallback-2971-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2648401 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52478_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-1-001/cores
[junit4] 2> 2648401 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52478_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2648405 INFO (OverseerStateUpdate-97351147139235845-127.0.0.1:52473_-n_0000000000) [n:127.0.0.1:52473_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2649426 WARN (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 2649427 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2649444 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2649550 WARN (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2649560 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2649581 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2649581 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 2649581 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d92275b
[junit4] 2> 2649586 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1672653504, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2649595 WARN (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2649613 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2649613 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2649614 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2649614 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2649615 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=55.8642578125, floorSegmentMB=0.7197265625, forceMergeDeletesPctAllowed=2.102139642005424, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2649616 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@30ea41ba[collection1] main]
[junit4] 2> 2649618 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2649619 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2649619 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2649620 INFO (searcherExecutor-6495-thread-1-processing-n:127.0.0.1:52478_ x:collection1 c:collection1) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@30ea41ba[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2649622 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557618363009597440
[junit4] 2> 2649637 INFO (coreZkRegister-6489-thread-1-processing-n:127.0.0.1:52478_ x:collection1 c:collection1) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2649637 INFO (coreZkRegister-6489-thread-1-processing-n:127.0.0.1:52478_ x:collection1 c:collection1) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2649637 INFO (coreZkRegister-6489-thread-1-processing-n:127.0.0.1:52478_ x:collection1 c:collection1) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:52478/collection1/
[junit4] 2> 2649637 INFO (coreZkRegister-6489-thread-1-processing-n:127.0.0.1:52478_ x:collection1 c:collection1) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2649638 INFO (coreZkRegister-6489-thread-1-processing-n:127.0.0.1:52478_ x:collection1 c:collection1) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy https://127.0.0.1:52478/collection1/ has no replicas
[junit4] 2> 2649652 INFO (coreZkRegister-6489-thread-1-processing-n:127.0.0.1:52478_ x:collection1 c:collection1) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:52478/collection1/ shard1
[junit4] 2> 2649816 INFO (coreZkRegister-6489-thread-1-processing-n:127.0.0.1:52478_ x:collection1 c:collection1) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2650264 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-2-001/cores/collection1
[junit4] 2> 2650265 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-2-001
[junit4] 2> 2650267 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2650270 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6819ca0{/,null,AVAILABLE}
[junit4] 2> 2650272 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@c974e59{SSL,[ssl, http/1.1]}{127.0.0.1:52496}
[junit4] 2> 2650272 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.Server Started @2655747ms
[junit4] 2> 2650273 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=52496, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-2-001/cores}
[junit4] 2> 2650274 ERROR (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2650274 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2650274 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2650274 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2650274 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-26T19:57:08.217Z
[junit4] 2> 2650278 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2650279 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-2-001/solr.xml
[junit4] 2> 2650291 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52443/solr
[junit4] 2> 2650309 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52496_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2650319 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52496_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52496_
[junit4] 2> 2650322 INFO (zkCallback-2962-thread-3-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2650322 INFO (zkCallback-2966-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2650322 INFO (zkCallback-2971-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2650323 INFO (zkCallback-2977-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2650459 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52496_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-2-001/cores
[junit4] 2> 2650459 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52496_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2650463 INFO (OverseerStateUpdate-97351147139235845-127.0.0.1:52473_-n_0000000000) [n:127.0.0.1:52473_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2651484 WARN (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 2651485 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2651498 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2651607 WARN (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2651610 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2651629 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2651630 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 2651630 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d92275b
[junit4] 2> 2651634 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1672653504, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2651672 WARN (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2651716 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2651716 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2651718 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2651718 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2651719 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=55.8642578125, floorSegmentMB=0.7197265625, forceMergeDeletesPctAllowed=2.102139642005424, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2651721 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@33ffcef2[collection1] main]
[junit4] 2> 2651723 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2651724 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2651724 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2651726 INFO (searcherExecutor-6506-thread-1-processing-n:127.0.0.1:52496_ x:collection1 c:collection1) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@33ffcef2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2651727 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557618365216849920
[junit4] 2> 2651734 INFO (coreZkRegister-6500-thread-1-processing-n:127.0.0.1:52496_ x:collection1 c:collection1) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 2651734 INFO (updateExecutor-2974-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2651735 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2651736 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2651736 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 2651736 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2651736 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [https://127.0.0.1:52478/collection1/] and I am [https://127.0.0.1:52496/collection1/]
[junit4] 2> 2651740 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [https://127.0.0.1:52478]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:52496_&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2651788 INFO (qtp339668053-15685) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 2651789 INFO (qtp339668053-15685) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 2651789 INFO (qtp339668053-15685) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:52496_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"https://127.0.0.1:52496","node_name":"127.0.0.1:52496_","state":"down"}
[junit4] 2> 2652471 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-3-001/cores/collection1
[junit4] 2> 2652472 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-3-001
[junit4] 2> 2652475 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2652478 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7985df93{/,null,AVAILABLE}
[junit4] 2> 2652478 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@29d19d84{SSL,[ssl, http/1.1]}{127.0.0.1:52506}
[junit4] 2> 2652478 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.Server Started @2657953ms
[junit4] 2> 2652480 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=52506, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-3-001/cores}
[junit4] 2> 2652481 ERROR (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2652481 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2652481 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2652481 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2652481 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-26T19:57:10.424Z
[junit4] 2> 2652486 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2652486 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-3-001/solr.xml
[junit4] 2> 2652514 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52443/solr
[junit4] 2> 2652533 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52506_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2652543 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52506_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52506_
[junit4] 2> 2652546 INFO (zkCallback-2971-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2652548 INFO (zkCallback-2984-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2652548 INFO (zkCallback-2977-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2652548 INFO (zkCallback-2962-thread-3-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2652548 INFO (zkCallback-2966-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2652668 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52506_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-3-001/cores
[junit4] 2> 2652668 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52506_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2652673 INFO (OverseerStateUpdate-97351147139235845-127.0.0.1:52473_-n_0000000000) [n:127.0.0.1:52473_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2652790 INFO (qtp339668053-15685) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:52496_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"https://127.0.0.1:52496","node_name":"127.0.0.1:52496_","state":"recovering"}
[junit4] 2> 2652790 INFO (qtp339668053-15685) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 2652790 INFO (qtp339668053-15685) [n:127.0.0.1:52478_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:52496_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1002
[junit4] 2> 2653695 WARN (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 2653696 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2653716 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2653853 WARN (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2653855 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2653884 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2653884 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 2653885 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d92275b
[junit4] 2> 2653888 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1672653504, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2653903 WARN (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2653928 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2653928 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2653929 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2653929 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2653931 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=55.8642578125, floorSegmentMB=0.7197265625, forceMergeDeletesPctAllowed=2.102139642005424, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2653932 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@721e0b05[collection1] main]
[junit4] 2> 2653934 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2653935 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2653935 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2653936 INFO (searcherExecutor-6517-thread-1-processing-n:127.0.0.1:52506_ x:collection1 c:collection1) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@721e0b05[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2653937 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557618367534202880
[junit4] 2> 2653946 INFO (coreZkRegister-6511-thread-1-processing-n:127.0.0.1:52506_ x:collection1 c:collection1) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 2653947 INFO (updateExecutor-2981-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2653948 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2653948 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2653949 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 2653949 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2653949 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [https://127.0.0.1:52478/collection1/] and I am [https://127.0.0.1:52506/collection1/]
[junit4] 2> 2653952 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [https://127.0.0.1:52478]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:52506_&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2653962 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 2653963 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 2653963 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:52506_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"https://127.0.0.1:52506","node_name":"127.0.0.1:52506_","state":"down"}
[junit4] 2> 2654101 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2654101 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30 for each attempt
[junit4] 2> 2654101 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
[junit4] 2> 2654971 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:52506_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"https://127.0.0.1:52506","node_name":"127.0.0.1:52506_","state":"recovering"}
[junit4] 2> 2654972 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 2654972 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:52506_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1009
[junit4] 2> 2659793 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [https://127.0.0.1:52478/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 2659793 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=https://127.0.0.1:52496 START replicas=[https://127.0.0.1:52478/collection1/] nUpdates=1000
[junit4] 2> 2659805 INFO (qtp339668053-15688) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2659805 INFO (qtp339668053-15688) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 2659807 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2659807 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2659807 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2659807 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2659808 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2659808 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2659808 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2659808 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2659808 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2661973 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [https://127.0.0.1:52478/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 2661973 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=https://127.0.0.1:52506 START replicas=[https://127.0.0.1:52478/collection1/] nUpdates=1000
[junit4] 2> 2661980 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2661980 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2661982 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2661982 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2661982 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2661982 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2661983 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2661983 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2661983 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2661983 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2661983 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2662115 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
[junit4] 2> 2662144 INFO (qtp846221081-15648) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2662144 INFO (qtp846221081-15648) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2662145 INFO (qtp846221081-15648) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2662145 INFO (qtp846221081-15648) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 1
[junit4] 2> 2662163 INFO (qtp339668053-15687) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2662163 INFO (qtp339668053-15687) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2662164 INFO (qtp339668053-15687) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2662164 INFO (qtp339668053-15687) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:52478/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 2662185 INFO (qtp463464142-15713) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2662185 INFO (qtp463464142-15713) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2662185 INFO (qtp878465336-15748) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2662186 INFO (qtp878465336-15748) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2662186 INFO (qtp463464142-15713) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2662186 INFO (qtp878465336-15748) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2662186 INFO (qtp878465336-15748) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:52478/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 2662186 INFO (qtp463464142-15713) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:52478/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 2662186 INFO (qtp339668053-15686) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 35
[junit4] 2> 2662194 INFO (qtp339668053-15685) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2662201 INFO (qtp463464142-15716) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2662210 INFO (qtp878465336-15751) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2664218 INFO (qtp846221081-15649) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1557618378312515584)} 0 3
[junit4] 2> 2664227 INFO (qtp463464142-15709) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1557618378316709888&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557618378316709888)} 0 2
[junit4] 2> 2664227 INFO (qtp878465336-15744) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1557618378316709888&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557618378316709888)} 0 2
[junit4] 2> 2664227 INFO (qtp339668053-15684) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1557618378316709888)} 0 7
[junit4] 2> 2664238 INFO (qtp878465336-15746) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[0 (1557618378330341376)]} 0 2
[junit4] 2> 2664238 INFO (qtp463464142-15711) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[0 (1557618378330341376)]} 0 3
[junit4] 2> 2664239 INFO (qtp339668053-15688) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[0 (1557618378330341376)]} 0 6
[junit4] 2> 2664241 INFO (qtp463464142-15712) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[1 (1557618378337681408)]} 0 0
[junit4] 2> 2664241 INFO (qtp878465336-15747) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[1 (1557618378337681408)]} 0 0
[junit4] 2> 2664242 INFO (qtp339668053-15687) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[1 (1557618378337681408)]} 0 2
[junit4] 2> 2664243 INFO (qtp463464142-15713) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[2 (1557618378339778560)]} 0 0
[junit4] 2> 2664243 INFO (qtp878465336-15748) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[2 (1557618378339778560)]} 0 0
[junit4] 2> 2664244 INFO (qtp339668053-15686) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[2 (1557618378339778560)]} 0 1
[junit4] 2> 2664253 INFO (qtp878465336-15750) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[3 (1557618378342924288)]} 0 0
[junit4] 2> 2664253 INFO (qtp463464142-15715) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[3 (1557618378342924288)]} 0 2
[junit4] 2> 2664254 INFO (qtp339668053-15681) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[3 (1557618378342924288)]} 0 8
[junit4] 2> 2664256 INFO (qtp463464142-15714) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[4 (1557618378353410048)]} 0 0
[junit4] 2> 2664256 INFO (qtp878465336-15750) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[4 (1557618378353410048)]} 0 0
[junit4] 2> 2664258 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[4 (1557618378353410048)]} 0 3
[junit4] 2> 2664260 INFO (qtp463464142-15716) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[5 (1557618378357604352)]} 0 0
[junit4] 2> 2664262 INFO (qtp878465336-15751) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[5 (1557618378357604352)]} 0 0
[junit4] 2> 2664263 INFO (qtp339668053-15685) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[5 (1557618378357604352)]} 0 4
[junit4] 2> 2664265 INFO (qtp878465336-15751) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[6 (1557618378362847232)]} 0 0
[junit4] 2> 2664266 INFO (qtp463464142-15716) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[6 (1557618378362847232)]} 0 0
[junit4] 2> 2664266 INFO (qtp339668053-15684) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[6 (1557618378362847232)]} 0 2
[junit4] 2> 2664269 INFO (qtp463464142-15711) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[7 (1557618378365992960)]} 0 0
[junit4] 2> 2664270 INFO (qtp878465336-15751) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= pa
[...truncated too long message...]
2855786 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 52506
[junit4] 2> 2855788 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52443 52443
[junit4] 2> 2855805 INFO (Thread-2880) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52443 52443
[junit4] 2> 2860650 WARN (Thread-2880) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 5 /solr/security.json
[junit4] 2> 5 /solr/configs/conf1
[junit4] 2> 4 /solr/collections/collection1/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 2 /solr/overseer_elect/election/97351147139235849-127.0.0.1:52478_-n_0000000001
[junit4] 2> 2 /solr/collections/collection1/leader_elect/shard1/election/97351147139235849-core_node1-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 209 /solr/overseer/collection-queue-work
[junit4] 2> 46 /solr/overseer/queue
[junit4] 2> 6 /solr/collections
[junit4] 2> 6 /solr/overseer/queue-work
[junit4] 2> 5 /solr/live_nodes
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=PeerSyncReplicationTest -Dtests.method=test -Dtests.seed=A0E5E2CB84D8E147 -Dtests.slow=true -Dtests.locale=nn-NO -Dtests.timezone=Australia/Broken_Hill -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] FAILURE 219s J1 | PeerSyncReplicationTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: timeout waiting to see all nodes active
[junit4] > at __randomizedtesting.SeedInfo.seed([A0E5E2CB84D8E147:28B1DD112A248CBF]:0)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2860694 INFO (SUITE-PeerSyncReplicationTest-seed#[A0E5E2CB84D8E147]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {other_tl1=PostingsFormat(name=Memory), range_facet_l_dv=FSTOrd50, rnd_s=PostingsFormat(name=LuceneVarGapDocFreqInterval), multiDefault=PostingsFormat(name=LuceneVarGapDocFreqInterval), a_t=PostingsFormat(name=Memory), intDefault=PostingsFormat(name=Memory), a_i1=PostingsFormat(name=Memory), id=FSTOrd50, range_facet_i_dv=PostingsFormat(name=Memory), text=Lucene50(blocksize=128), range_facet_l=PostingsFormat(name=Memory), timestamp=PostingsFormat(name=Memory)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Lucene70), _version_=DocValuesFormat(name=Memory), range_facet_i_dv=DocValuesFormat(name=Memory), intDvoDefault=DocValuesFormat(name=Direct), timestamp=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1706, maxMBSortInHeap=5.210213964200542, sim=RandomSimilarity(queryNorm=false): {}, locale=nn-NO, timezone=Australia/Broken_Hill
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=224744368,total=530579456
[junit4] 2> NOTE: All tests run in this JVM: [TestHdfsUpdateLog, TemplateUpdateProcessorTest, TestOverriddenPrefixQueryForCustomFieldType, TestSolr4Spatial, ExternalFileFieldSortTest, ReturnFieldsTest, SuggesterWFSTTest, ResponseLogComponentTest, TestManagedResource, InfoHandlerTest, PreAnalyzedUpdateProcessorTest, TestOnReconnectListenerSupport, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestPHPSerializedResponseWriter, DateRangeFieldTest, BlockJoinFacetDistribTest, TestDistributedMissingSort, TestDocBasedVersionConstraints, BooleanFieldTest, TestMinMaxOnMultiValuedField, TestLuceneMatchVersion, TestBinaryResponseWriter, TestFoldingMultitermQuery, TestJsonFacetRefinement, TestSolrCloudWithKerberosAlt, HdfsUnloadDistributedZkTest, TestDistributedGrouping, MissingSegmentRecoveryTest, TestExclusionRuleCollectionAccess, TestOmitPositions, LeaderInitiatedRecoveryOnShardRestartTest, TestStressUserVersions, HighlighterConfigTest, SSLMigrationTest, StatsReloadRaceTest, TestSolrQueryParser, SortSpecParsingTest, MinimalSchemaTest, RequestHandlersTest, TestReversedWildcardFilterFactory, TestSortByMinMaxFunction, TestDelegationWithHadoopAuth, TestCrossCoreJoin, ConfigSetsAPITest, SolrJmxReporterTest, TestSuggestSpellingConverter, SolrCoreMetricManagerTest, TestRestManager, RegexBoostProcessorTest, LukeRequestHandlerTest, QueryResultKeyTest, TolerantUpdateProcessorTest, SignatureUpdateProcessorFactoryTest, TestEmbeddedSolrServerConstructors, OverseerTaskQueueTest, TestExportWriter, TestWriterPerf, TestFieldCacheSort, TestDefaultStatsCache, TestSizeLimitedDistributedMap, DistributedFacetPivotLargeTest, ShowFileRequestHandlerTest, CloneFieldUpdateProcessorFactoryTest, TestCodecSupport, CollectionReloadTest, TestCorePropertiesReload, TestHashQParserPlugin, TestCollectionAPI, TermVectorComponentDistributedTest, TestFiltering, TestGraphMLResponseWriter, TestRuleBasedAuthorizationPlugin, MergeStrategyTest, TestFilteredDocIdSet, TestFieldCacheVsDocValues, TestSimpleQParserPlugin, TestSegmentSorting, ClassificationUpdateProcessorIntegrationTest, TestRTimerTree, DebugComponentTest, BasicAuthStandaloneTest, AutoCommitTest, QueryParsingTest, TestReplicaProperties, AddBlockUpdateTest, TestConfigSetsAPIExclusivity, DistributedQueryComponentOptimizationTest, SolrXmlInZkTest, TestSearchPerf, JavabinLoaderTest, URLClassifyProcessorTest, RulesTest, ChaosMonkeySafeLeaderTest, TestSolrDynamicMBean, BlockDirectoryTest, TriLevelCompositeIdRoutingTest, TestReloadDeadlock, JsonLoaderTest, TestCustomSort, TestJsonRequest, TestPushWriter, TestInPlaceUpdatesStandalone, TestRestoreCore, TestPointFields, UUIDFieldTest, HdfsBasicDistributedZk2Test, TestExactSharedStatsCache, TestFuzzyAnalyzedSuggestions, SpellPossibilityIteratorTest, FullHLLTest, TestFieldCacheReopen, TestLegacyNumericRangeQueryBuilder, DistributedFacetPivotSmallAdvancedTest, NotRequiredUniqueKeyTest, TestCryptoKeys, HdfsRecoverLeaseTest, TestStressLiveNodes, HdfsThreadLeakTest, SuggestComponentContextFilterQueryTest, UpdateParamsTest, JSONWriterTest, ScriptEngineTest, DistribJoinFromCollectionTest, TestSimpleTrackingShardHandler, TestLMJelinekMercerSimilarityFactory, TestRandomCollapseQParserPlugin, HLLUtilTest, RecoveryAfterSoftCommitTest, TestAuthorizationFramework, DirectUpdateHandlerOptimizeTest, TestSchemaVersionResource, OpenCloseCoreStressTest, TestJmxMonitoredMap, ForceLeaderTest, TestCloudDeleteByQuery, CollectionTooManyReplicasTest, TestStressCloudBlindAtomicUpdates, DeleteNodeTest, IndexSchemaTest, TestSolrFieldCacheMBean, TestStressRecovery, FileBasedSpellCheckerTest, TestShardHandlerFactory, TestDistribDocBasedVersion, TestConfigSetProperties, SimpleMLTQParserTest, AliasIntegrationTest, StressHdfsTest, TestSubQueryTransformerCrossCore, FieldAnalysisRequestHandlerTest, TestAnalyzedSuggestions, TestSolrCoreProperties, DistribCursorPagingTest, SpellCheckCollatorTest, TestSolrDeletionPolicy2, BadIndexSchemaTest, TestHighFrequencyDictionaryFactory, TestTestInjection, ResourceLoaderTest, CircularListTest, CdcrReplicationHandlerTest, CdcrVersionReplicationTest, BigEndianAscendingWordDeserializerTest, TestSolrConfigHandlerConcurrent, BasicDistributedZk2Test, CollectionsAPIDistributedZkTest, LeaderElectionIntegrationTest, ClusterStateUpdateTest, TestRandomFaceting, ShardRoutingCustomTest, TestDistributedSearch, ZkControllerTest, TestStressReorder, TestJoin, TestStressVersions, HardAutoCommitTest, TestRangeQuery, SuggesterTSTTest, SuggesterTest, TestTrie, NoCacheHeaderTest, WordBreakSolrSpellCheckerTest, SolrCoreCheckLockOnStartupTest, FieldMutatingUpdateProcessorTest, TestAtomicUpdateErrorCases, DefaultValueUpdateProcessorTest, SortByFunctionTest, TestRemoteStreaming, DocValuesMultiTest, DistanceFunctionTest, SolrInfoMBeanTest, TestQueryUtils, DirectSolrSpellCheckerTest, PrimitiveFieldTypeTest, XmlUpdateRequestHandlerTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, FastVectorHighlighterTest, LoggingHandlerTest, TestCollationField, TestCSVResponseWriter, BinaryUpdateRequestHandlerTest, PingRequestHandlerTest, TestPhraseSuggestions, TestXIncludeConfig, TestSweetSpotSimilarityFactory, TestLMDirichletSimilarityFactory, TestFastWriter, TestFastLRUCache, PreAnalyzedFieldTest, DateFieldTest, ClusterStateTest, TestUtils, TestRTGBase, CursorPagingTest, CdcrRequestHandlerTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, CollectionStateFormat2Test, CollectionsAPISolrJTest, DeleteInactiveReplicaTest, DistributedQueueTest, DocValuesNotIndexedTest, HttpPartitionTest, OverseerCollectionConfigSetProcessorTest, OverseerRolesTest, PeerSyncReplicationTest]
[junit4] Completed [506/684 (1!)] on J1 in 219.49s, 1 test, 1 failure <<< FAILURES!
[...truncated 63041 lines...]
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3799
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3799/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
1 tests failed.
FAILED: org.apache.solr.cloud.PeerSyncReplicationTest.test
Error Message:
timeout waiting to see all nodes active
Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
at __randomizedtesting.SeedInfo.seed([ECCB97A5D970679F:649FA87F778C0A67]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12315 lines...]
[junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/init-core-data-001
[junit4] 2> 2687799 INFO (SUITE-PeerSyncReplicationTest-seed#[ECCB97A5D970679F]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields
[junit4] 2> 2687802 INFO (SUITE-PeerSyncReplicationTest-seed#[ECCB97A5D970679F]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 2687802 INFO (SUITE-PeerSyncReplicationTest-seed#[ECCB97A5D970679F]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /i/d
[junit4] 2> 2687807 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2687808 INFO (Thread-3425) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2687808 INFO (Thread-3425) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2687910 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.ZkTestServer start zk server on port:62260
[junit4] 2> 2687941 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 2687957 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 2687962 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 2687966 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 2687970 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 2687973 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 2687976 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 2687980 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 2687983 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 2687986 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 2687990 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 2689690 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/control-001/cores/collection1
[junit4] 2> 2689693 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2689696 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@163a0918{/i/d,null,AVAILABLE}
[junit4] 2> 2689696 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2d1ad56a{HTTP/1.1,[http/1.1]}{127.0.0.1:62263}
[junit4] 2> 2689696 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.Server Started @2695624ms
[junit4] 2> 2689697 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/tempDir-001/control/data, hostContext=/i/d, hostPort=62263, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/control-001/cores}
[junit4] 2> 2689697 ERROR (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2689698 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2689698 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2689698 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2689698 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-26T09:27:36.925Z
[junit4] 2> 2689703 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2689703 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/control-001/solr.xml
[junit4] 2> 2689730 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:62260/solr
[junit4] 2> 2689779 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:62263_i%2Fd
[junit4] 2> 2689781 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.Overseer Overseer (id=97348672227704836-127.0.0.1:62263_i%2Fd-n_0000000000) starting
[junit4] 2> 2689795 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62263_i%2Fd
[junit4] 2> 2689798 INFO (OverseerStateUpdate-97348672227704836-127.0.0.1:62263_i%2Fd-n_0000000000) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2689839 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/control-001/cores
[junit4] 2> 2689839 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2689844 INFO (OverseerStateUpdate-97348672227704836-127.0.0.1:62263_i%2Fd-n_0000000000) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2690861 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2690878 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2690967 WARN (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2690969 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2690989 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 2690989 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/control-001/cores/collection1/data/]
[junit4] 2> 2690989 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@9f078b
[junit4] 2> 2690994 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=2065311905, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2691038 WARN (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2691057 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2691058 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2691059 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2691059 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2691059 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=5.5322265625, floorSegmentMB=1.2119140625, forceMergeDeletesPctAllowed=3.699016373460373, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5180719917556548
[junit4] 2> 2691060 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5db329f0[collection1] main]
[junit4] 2> 2691062 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2691063 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2691063 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2691064 INFO (searcherExecutor-7724-thread-1-processing-n:127.0.0.1:62263_i%2Fd x:collection1 c:control_collection) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5db329f0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2691065 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557578759056392192
[junit4] 2> 2691075 INFO (coreZkRegister-7716-thread-1-processing-n:127.0.0.1:62263_i%2Fd x:collection1 c:control_collection) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2691075 INFO (coreZkRegister-7716-thread-1-processing-n:127.0.0.1:62263_i%2Fd x:collection1 c:control_collection) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2691075 INFO (coreZkRegister-7716-thread-1-processing-n:127.0.0.1:62263_i%2Fd x:collection1 c:control_collection) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:62263/i/d/collection1/
[junit4] 2> 2691075 INFO (coreZkRegister-7716-thread-1-processing-n:127.0.0.1:62263_i%2Fd x:collection1 c:control_collection) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2691075 INFO (coreZkRegister-7716-thread-1-processing-n:127.0.0.1:62263_i%2Fd x:collection1 c:control_collection) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:62263/i/d/collection1/ has no replicas
[junit4] 2> 2691083 INFO (coreZkRegister-7716-thread-1-processing-n:127.0.0.1:62263_i%2Fd x:collection1 c:control_collection) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:62263/i/d/collection1/ shard1
[junit4] 2> 2691118 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2691120 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:62260/solr ready
[junit4] 2> 2691120 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 2691120 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
[junit4] 2> 2691136 INFO (coreZkRegister-7716-thread-1-processing-n:127.0.0.1:62263_i%2Fd x:collection1 c:control_collection) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2691546 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-1-001/cores/collection1
[junit4] 2> 2691547 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-1-001
[junit4] 2> 2691548 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2691550 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3da55be9{/i/d,null,AVAILABLE}
[junit4] 2> 2691550 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3ab552af{HTTP/1.1,[http/1.1]}{127.0.0.1:62269}
[junit4] 2> 2691550 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.Server Started @2697479ms
[junit4] 2> 2691550 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/i/d, hostPort=62269, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-1-001/cores}
[junit4] 2> 2691552 ERROR (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2691552 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2691552 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2691552 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2691552 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-26T09:27:38.779Z
[junit4] 2> 2691556 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2691556 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-1-001/solr.xml
[junit4] 2> 2691624 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:62260/solr
[junit4] 2> 2691640 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2691650 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62269_i%2Fd
[junit4] 2> 2691653 INFO (zkCallback-3132-thread-3-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2691653 INFO (zkCallback-3136-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2691654 INFO (zkCallback-3142-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2691681 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-1-001/cores
[junit4] 2> 2691682 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2691685 INFO (OverseerStateUpdate-97348672227704836-127.0.0.1:62263_i%2Fd-n_0000000000) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2691795 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2692710 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2692726 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2692800 WARN (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2692802 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2692822 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2692822 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 2692822 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@9f078b
[junit4] 2> 2692836 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=2065311905, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2692859 WARN (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2692879 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2692879 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2692880 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2692880 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2692881 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=5.5322265625, floorSegmentMB=1.2119140625, forceMergeDeletesPctAllowed=3.699016373460373, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5180719917556548
[junit4] 2> 2692882 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@a53aadd[collection1] main]
[junit4] 2> 2692883 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2692884 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2692884 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2692885 INFO (searcherExecutor-7735-thread-1-processing-n:127.0.0.1:62269_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@a53aadd[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2692886 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557578760965849088
[junit4] 2> 2692896 INFO (coreZkRegister-7729-thread-1-processing-n:127.0.0.1:62269_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2692897 INFO (coreZkRegister-7729-thread-1-processing-n:127.0.0.1:62269_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2692897 INFO (coreZkRegister-7729-thread-1-processing-n:127.0.0.1:62269_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:62269/i/d/collection1/
[junit4] 2> 2692897 INFO (coreZkRegister-7729-thread-1-processing-n:127.0.0.1:62269_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2692897 INFO (coreZkRegister-7729-thread-1-processing-n:127.0.0.1:62269_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:62269/i/d/collection1/ has no replicas
[junit4] 2> 2692904 INFO (coreZkRegister-7729-thread-1-processing-n:127.0.0.1:62269_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:62269/i/d/collection1/ shard1
[junit4] 2> 2693016 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2693066 INFO (coreZkRegister-7729-thread-1-processing-n:127.0.0.1:62269_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2693174 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2693278 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-2-001/cores/collection1
[junit4] 2> 2693280 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-2-001
[junit4] 2> 2693281 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2693283 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@43204441{/i/d,null,AVAILABLE}
[junit4] 2> 2693283 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2673b2f3{HTTP/1.1,[http/1.1]}{127.0.0.1:62276}
[junit4] 2> 2693283 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.Server Started @2699212ms
[junit4] 2> 2693283 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/i/d, hostPort=62276, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-2-001/cores}
[junit4] 2> 2693284 ERROR (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2693284 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2693284 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2693284 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2693284 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-26T09:27:40.511Z
[junit4] 2> 2693290 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2693290 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-2-001/solr.xml
[junit4] 2> 2693310 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:62260/solr
[junit4] 2> 2693329 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2693339 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62276_i%2Fd
[junit4] 2> 2693352 INFO (zkCallback-3136-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2693353 INFO (zkCallback-3132-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2693354 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2693354 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2693385 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-2-001/cores
[junit4] 2> 2693385 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2693390 INFO (OverseerStateUpdate-97348672227704836-127.0.0.1:62263_i%2Fd-n_0000000000) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2693495 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2693495 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2694403 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2694421 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2694498 WARN (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2694499 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2694519 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2694520 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 2694520 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@9f078b
[junit4] 2> 2694523 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=2065311905, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2694534 WARN (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2694555 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2694555 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2694556 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2694556 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2694557 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=5.5322265625, floorSegmentMB=1.2119140625, forceMergeDeletesPctAllowed=3.699016373460373, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5180719917556548
[junit4] 2> 2694557 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3a7267f[collection1] main]
[junit4] 2> 2694559 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2694560 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2694560 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2694561 INFO (searcherExecutor-7746-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3a7267f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2694562 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557578762723262464
[junit4] 2> 2694568 INFO (coreZkRegister-7740-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 2694568 INFO (updateExecutor-3145-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2694568 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2694568 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2694569 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 2694569 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2694569 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:62269/i/d/collection1/] and I am [http://127.0.0.1:62276/i/d/collection1/]
[junit4] 2> 2694571 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:62269/i/d]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:62276_i%252Fd&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2694575 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 2694575 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 2694575 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:62276_i%2Fd, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:62276/i/d","node_name":"127.0.0.1:62276_i%2Fd","state":"down"}
[junit4] 2> 2694686 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2694686 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2695019 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-3-001/cores/collection1
[junit4] 2> 2695020 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-3-001
[junit4] 2> 2695021 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2695023 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1a894007{/i/d,null,AVAILABLE}
[junit4] 2> 2695023 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@41112563{HTTP/1.1,[http/1.1]}{127.0.0.1:62286}
[junit4] 2> 2695023 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.Server Started @2700952ms
[junit4] 2> 2695023 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/i/d, hostPort=62286, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-3-001/cores}
[junit4] 2> 2695024 ERROR (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2695025 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2695025 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2695025 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2695025 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-26T09:27:42.252Z
[junit4] 2> 2695029 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2695030 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-3-001/solr.xml
[junit4] 2> 2695051 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:62260/solr
[junit4] 2> 2695068 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2695077 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62286_i%2Fd
[junit4] 2> 2695080 INFO (zkCallback-3136-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2695080 INFO (zkCallback-3132-thread-3-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2695080 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2695081 INFO (zkCallback-3155-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2695081 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2695145 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-3-001/cores
[junit4] 2> 2695145 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2695148 INFO (OverseerStateUpdate-97348672227704836-127.0.0.1:62263_i%2Fd-n_0000000000) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2695254 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2695254 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2695254 INFO (zkCallback-3155-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2695577 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:62276_i%2Fd, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:62276/i/d","node_name":"127.0.0.1:62276_i%2Fd","state":"recovering"}
[junit4] 2> 2695578 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 2695578 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:62276_i%252Fd&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1002
[junit4] 2> 2696170 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2696188 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2696263 WARN (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2696265 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2696284 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2696284 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 2696284 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@9f078b
[junit4] 2> 2696288 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=2065311905, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2696316 WARN (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2696334 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2696334 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2696335 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2696335 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2696337 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=5.5322265625, floorSegmentMB=1.2119140625, forceMergeDeletesPctAllowed=3.699016373460373, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5180719917556548
[junit4] 2> 2696338 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@ad1b8a5[collection1] main]
[junit4] 2> 2696339 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2696340 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2696340 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2696341 INFO (searcherExecutor-7757-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@ad1b8a5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2696342 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557578764589727744
[junit4] 2> 2696348 INFO (coreZkRegister-7751-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 2696348 INFO (updateExecutor-3152-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2696349 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2696349 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2696349 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 2696349 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2696349 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:62269/i/d/collection1/] and I am [http://127.0.0.1:62286/i/d/collection1/]
[junit4] 2> 2696352 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:62269/i/d]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:62286_i%252Fd&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2696353 INFO (qtp1543301162-17464) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 2696354 INFO (qtp1543301162-17464) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 2696354 INFO (qtp1543301162-17464) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:62286_i%2Fd, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:62286/i/d","node_name":"127.0.0.1:62286_i%2Fd","state":"down"}
[junit4] 2> 2696455 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2696456 INFO (zkCallback-3155-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2696455 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2696506 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2696506 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30 for each attempt
[junit4] 2> 2696506 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
[junit4] 2> 2697354 INFO (qtp1543301162-17464) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:62286_i%2Fd, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:62286/i/d","node_name":"127.0.0.1:62286_i%2Fd","state":"recovering"}
[junit4] 2> 2697354 INFO (qtp1543301162-17464) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 2697354 INFO (qtp1543301162-17464) [n:127.0.0.1:62269_i%2Fd ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:62286_i%252Fd&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1001
[junit4] 2> 2702584 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:62269/i/d/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 2702585 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:62276/i/d START replicas=[http://127.0.0.1:62269/i/d/collection1/] nUpdates=1000
[junit4] 2> 2702587 INFO (qtp1543301162-17461) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2702587 INFO (qtp1543301162-17461) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp=/i/d path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2702588 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2702588 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2702588 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2702588 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2702589 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2702589 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2702589 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2702589 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2702589 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2702594 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2702594 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2702594 INFO (zkCallback-3155-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2704359 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:62269/i/d/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 2704360 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:62286/i/d START replicas=[http://127.0.0.1:62269/i/d/collection1/] nUpdates=1000
[junit4] 2> 2704362 INFO (qtp1543301162-17462) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2704362 INFO (qtp1543301162-17462) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp=/i/d path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2704363 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2704363 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2704363 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2704363 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2704363 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2704364 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2704364 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2704364 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2704364 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2704375 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2704375 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2704375 INFO (zkCallback-3155-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2704539 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
[junit4] 2> 2704542 INFO (qtp1875057789-17421) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2704542 INFO (qtp1875057789-17421) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2704543 INFO (qtp1875057789-17421) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2704544 INFO (qtp1875057789-17421) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/i/d path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 1
[junit4] 2> 2704547 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2704547 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2704549 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2704550 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/i/d path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:62269/i/d/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 2
[junit4] 2> 2704553 INFO (qtp515180737-17526) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2704553 INFO (qtp515180737-17526) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2704553 INFO (qtp937108718-17491) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2704554 INFO (qtp937108718-17491) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2704555 INFO (qtp515180737-17526) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2704555 INFO (qtp515180737-17526) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/i/d path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:62269/i/d/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 5
[junit4] 2> 2704555 INFO (qtp937108718-17491) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2704555 INFO (qtp937108718-17491) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/i/d path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:62269/i/d/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 2
[junit4] 2> 2704556 INFO (qtp1543301162-17465) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/i/d path=/update params={_stateVer_=collection1:9&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 10
[junit4] 2> 2704559 INFO (qtp1543301162-17461) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp=/i/d path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2704561 INFO (qtp937108718-17493) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.S.Request [collection1] webapp=/i/d path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2704562 INFO (qtp515180737-17528) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.S.Request [collection1] webapp=/i/d path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2706572 INFO (qtp1875057789-17422) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/i/d path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1557578775313514496)} 0 2
[junit4] 2> 2706577 INFO (qtp937108718-17494) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/i/d path=/update params={
[...truncated too long message...]
2 WARN (Thread-3425) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 5 /solr/security.json
[junit4] 2> 5 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 4 /solr/collections/collection1/state.json
[junit4] 2> 2 /solr/collections/collection1/leader_elect/shard1/election/97348672227704841-core_node1-n_0000000000
[junit4] 2> 2 /solr/overseer_elect/election/97348672227704841-127.0.0.1:62269_i%2Fd-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 207 /solr/overseer/collection-queue-work
[junit4] 2> 43 /solr/overseer/queue
[junit4] 2> 14 /solr/overseer/queue-work
[junit4] 2> 6 /solr/collections
[junit4] 2> 5 /solr/live_nodes
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=PeerSyncReplicationTest -Dtests.method=test -Dtests.seed=ECCB97A5D970679F -Dtests.slow=true -Dtests.locale=ko-KR -Dtests.timezone=Asia/Seoul -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 215s J1 | PeerSyncReplicationTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: timeout waiting to see all nodes active
[junit4] > at __randomizedtesting.SeedInfo.seed([ECCB97A5D970679F:649FA87F778C0A67]:0)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2902995 INFO (SUITE-PeerSyncReplicationTest-seed#[ECCB97A5D970679F]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {other_tl1=PostingsFormat(name=Memory), range_facet_l_dv=PostingsFormat(name=LuceneVarGapFixedInterval), rnd_s=PostingsFormat(name=Memory), multiDefault=PostingsFormat(name=Memory), a_t=PostingsFormat(name=Memory), intDefault=PostingsFormat(name=Memory), a_i1=PostingsFormat(name=Memory), id=PostingsFormat(name=LuceneVarGapFixedInterval), range_facet_i_dv=PostingsFormat(name=Memory), text=FSTOrd50, range_facet_l=PostingsFormat(name=Memory), timestamp=PostingsFormat(name=Memory)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Memory), _version_=DocValuesFormat(name=Lucene70), range_facet_i_dv=DocValuesFormat(name=Lucene70), intDvoDefault=DocValuesFormat(name=Lucene70), timestamp=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=987, maxMBSortInHeap=5.369901637346038, sim=RandomSimilarity(queryNorm=true): {}, locale=ko-KR, timezone=Asia/Seoul
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=113352272,total=363687936
[junit4] 2> NOTE: All tests run in this JVM: [TestNRTOpen, OverseerTaskQueueTest, TestLMDirichletSimilarityFactory, TestSolrQueryParserDefaultOperatorResource, NotRequiredUniqueKeyTest, UpdateParamsTest, TestExportWriter, ShufflingReplicaListTransformerTest, TestRangeQuery, OutOfBoxZkACLAndCredentialsProvidersTest, DirectUpdateHandlerOptimizeTest, SpellCheckCollatorTest, SSLMigrationTest, TestUseDocValuesAsStored2, TestSolrQueryParser, BitVectorTest, DeleteInactiveReplicaTest, TestFoldingMultitermQuery, CollectionsAPISolrJTest, TestWriterPerf, VersionInfoTest, TestRestManager, TestCoreDiscovery, InfoHandlerTest, RequestLoggingTest, TestShardHandlerFactory, TestDocTermOrds, SolrXmlInZkTest, TestIndexSearcher, TestCollectionAPI, TestSQLHandlerNonCloud, TestStressRecovery, JSONWriterTest, DistribCursorPagingTest, TestSolrDeletionPolicy2, CdcrUpdateLogTest, LeaderInitiatedRecoveryOnShardRestartTest, TestManagedSynonymFilterFactory, TestEmbeddedSolrServerConstructors, SuggesterTSTTest, ShardRoutingCustomTest, LoggingHandlerTest, TestCorePropertiesReload, TestSegmentSorting, TestScoreJoinQPNoScore, CreateCollectionCleanupTest, MinimalSchemaTest, DateMathParserTest, TestCloudPivotFacet, TestDistributedStatsComponentCardinality, TestSimpleTrackingShardHandler, ResponseLogComponentTest, TestDynamicFieldResource, TestLegacyFieldCache, TestQuerySenderListener, CoreSorterTest, BadIndexSchemaTest, PreAnalyzedUpdateProcessorTest, TestComponentsName, TestAnalyzedSuggestions, TestPointFields, TestJsonRequest, TestFieldCacheSort, AtomicUpdatesTest, DistributedFacetPivotLargeTest, OverseerCollectionConfigSetProcessorTest, DistributedIntervalFacetingTest, FieldAnalysisRequestHandlerTest, TestRTimerTree, DistributedVersionInfoTest, DeleteShardTest, TestDFISimilarityFactory, AutoCommitTest, TestLRUStatsCache, MissingSegmentRecoveryTest, BasicZkTest, TestZkChroot, DistributedTermsComponentTest, TestFileDictionaryLookup, TestManagedResource, TestSearchPerf, WrapperMergePolicyFactoryTest, SolrCoreMetricManagerTest, TestGraphMLResponseWriter, CircularListTest, SortByFunctionTest, TestOverriddenPrefixQueryForCustomFieldType, ScriptEngineTest, TestSolrConfigHandlerConcurrent, EnumFieldTest, TestCloudRecovery, LeaderFailureAfterFreshStartTest, TestRawTransformer, PeerSyncWithIndexFingerprintCachingTest, TestAuthorizationFramework, TestSchemalessBufferedUpdates, BlockJoinFacetDistribTest, DateRangeFieldTest, TestSubQueryTransformerCrossCore, SimpleMLTQParserTest, TestDefaultStatsCache, TestHighFrequencyDictionaryFactory, TestFieldCacheVsDocValues, SolrCLIZkUtilsTest, SignatureUpdateProcessorFactoryTest, DistributedFacetPivotSmallAdvancedTest, ConvertedLegacyTest, SpellingQueryConverterTest, OpenCloseCoreStressTest, HLLUtilTest, ReturnFieldsTest, UpdateRequestProcessorFactoryTest, SpellCheckComponentTest, TestSolrCloudWithHadoopAuthPlugin, TestStressLiveNodes, TriLevelCompositeIdRoutingTest, SolrMetricReporterTest, BooleanFieldTest, CacheHeaderTest, SolrGangliaReporterTest, RulesTest, TestUniqueKeyFieldResource, QueryResultKeyTest, TestFieldResource, AliasIntegrationTest, TestSolrCloudWithKerberosAlt, TestDistribDocBasedVersion, TestCustomSort, TestLMJelinekMercerSimilarityFactory, DistribJoinFromCollectionTest, CdcrReplicationHandlerTest, TestReplicaProperties, TestRandomCollapseQParserPlugin, DefaultValueUpdateProcessorTest, AnalyticsQueryTest, TestReversedWildcardFilterFactory, URLClassifyProcessorTest, TestFieldTypeResource, SolrCmdDistributorTest, TestSolrCoreProperties, PreAnalyzedFieldManagedSchemaCloudTest, BlockDirectoryTest, ShardSplitTest, NoCacheHeaderTest, CollectionReloadTest, ZkSolrClientTest, HdfsRecoverLeaseTest, ConfigSetsAPITest, CleanupOldIndexTest, CurrencyFieldOpenExchangeTest, TestDelegationWithHadoopAuth, StatsComponentTest, SolrIndexSplitterTest, TestSystemIdResolver, TestDocBasedVersionConstraints, IndexSchemaTest, HighlighterConfigTest, TestExactSharedStatsCache, DebugComponentTest, TestUtils, TestUnifiedSolrHighlighter, TestSortByMinMaxFunction, DistributedExpandComponentTest, HdfsRestartWhileUpdatingTest, JavabinLoaderTest, ShardRoutingTest, TestRandomFaceting, TestRandomDVFaceting, TestDistributedSearch, DistributedSpellCheckComponentTest, TestRealTimeGet, TestJoin, HardAutoCommitTest, QueryElevationComponentTest, TestSort, TestFiltering, TestFunctionQuery, TestLazyCores, SoftAutoCommitTest, CurrencyFieldXmlFileTest, SimplePostToolTest, TestExtendedDismaxParser, SolrRequestParserTest, SuggesterTest, TestStressLucene, TestTrie, SpatialFilterTest, PolyFieldTest, WordBreakSolrSpellCheckerTest, SolrCoreCheckLockOnStartupTest, TestAtomicUpdateErrorCases, TestRemoteStreaming, SolrInfoMBeanTest, LukeRequestHandlerTest, FileBasedSpellCheckerTest, XmlUpdateRequestHandlerTest, RequestHandlersTest, TestValueSourceCache, TestIndexingPerformance, TestCollationField, TestCSVResponseWriter, UniqFieldsUpdateProcessorFactoryTest, TestPartialUpdateDeduplication, CSVRequestHandlerTest, AlternateDirectoryTest, TestSolrIndexConfig, CopyFieldTest, SolrIndexConfigTest, BadComponentTest, TestMergePolicyConfig, TestDocSet, NumericFieldsTest, TestElisionMultitermQuery, TestConfig, OutputWriterTest, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, TestPostingsSolrHighlighter, TestLuceneMatchVersion, TestPhraseSuggestions, SpellPossibilityIteratorTest, TestCodecSupport, SynonymTokenizerTest, EchoParamsTest, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestIBSimilarityFactory, TestBM25SimilarityFactory, TestJmxMonitoredMap, TestFastLRUCache, TestSuggestSpellingConverter, RAMDirectoryFactoryTest, SliceStateTest, SystemInfoHandlerTest, UUIDFieldTest, CursorPagingTest, CdcrRequestHandlerTest, CdcrVersionReplicationTest, CollectionStateFormat2Test, CollectionsAPIAsyncDistributedZkTest, DeleteLastCustomShardedReplicaTest, DeleteNodeTest, DeleteStatusTest, DistributedQueueTest, ForceLeaderTest, LeaderElectionContextKeyTest, LeaderFailoverAfterPartitionTest, OverriddenZkACLAndCredentialsProvidersTest, PeerSyncReplicationTest]
[junit4] Completed [518/684 (1!)] on J1 in 215.22s, 1 test, 1 failure <<< FAILURES!
[...truncated 62983 lines...]
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3798
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3798/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC
1 tests failed.
FAILED: org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingSorting
Error Message:
Should have exactly 4 documents returned expected:<4> but was:<3>
Stack Trace:
java.lang.AssertionError: Should have exactly 4 documents returned expected:<4> but was:<3>
at __randomizedtesting.SeedInfo.seed([36B7C7B063946B0B:288FCFB81F3FD18B]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.apache.solr.cloud.DocValuesNotIndexedTest.checkSortOrder(DocValuesNotIndexedTest.java:259)
at org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingSorting(DocValuesNotIndexedTest.java:244)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12209 lines...]
[junit4] Suite: org.apache.solr.cloud.DocValuesNotIndexedTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/init-core-data-001
[junit4] 2> 2402936 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields
[junit4] 2> 2402943 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 2402945 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001
[junit4] 2> 2402945 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2402946 INFO (Thread-3490) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2402946 INFO (Thread-3490) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2403051 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:53916
[junit4] 2> 2403260 INFO (jetty-launcher-2861-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2403262 INFO (jetty-launcher-2861-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2403264 INFO (jetty-launcher-2861-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6417bf1d{/solr,null,AVAILABLE}
[junit4] 2> 2403264 INFO (jetty-launcher-2861-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5962d615{HTTP/1.1,[http/1.1]}{127.0.0.1:53918}
[junit4] 2> 2403264 INFO (jetty-launcher-2861-thread-1) [ ] o.e.j.s.Server Started @2433856ms
[junit4] 2> 2403264 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=53918}
[junit4] 2> 2403264 ERROR (jetty-launcher-2861-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2403265 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2403265 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2403265 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2403265 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-25T23:45:08.636Z
[junit4] 2> 2403269 INFO (jetty-launcher-2861-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@574b78af{/solr,null,AVAILABLE}
[junit4] 2> 2403269 INFO (jetty-launcher-2861-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7ab1375e{HTTP/1.1,[http/1.1]}{127.0.0.1:53919}
[junit4] 2> 2403270 INFO (jetty-launcher-2861-thread-2) [ ] o.e.j.s.Server Started @2433862ms
[junit4] 2> 2403270 INFO (jetty-launcher-2861-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=53919}
[junit4] 2> 2403270 ERROR (jetty-launcher-2861-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2403271 INFO (jetty-launcher-2861-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2403271 INFO (jetty-launcher-2861-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2403271 INFO (jetty-launcher-2861-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2403271 INFO (jetty-launcher-2861-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-25T23:45:08.642Z
[junit4] 2> 2403280 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2403281 INFO (jetty-launcher-2861-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2403293 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53916/solr
[junit4] 2> 2403297 INFO (jetty-launcher-2861-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53916/solr
[junit4] 2> 2403367 INFO (jetty-launcher-2861-thread-2) [n:127.0.0.1:53919_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:53919_solr
[junit4] 2> 2403369 INFO (jetty-launcher-2861-thread-1) [n:127.0.0.1:53918_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53918_solr
[junit4] 2> 2403370 INFO (jetty-launcher-2861-thread-2) [n:127.0.0.1:53919_solr ] o.a.s.c.Overseer Overseer (id=97346381959790598-127.0.0.1:53919_solr-n_0000000000) starting
[junit4] 2> 2403373 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2403374 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2403393 INFO (jetty-launcher-2861-thread-2) [n:127.0.0.1:53919_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53919_solr
[junit4] 2> 2403396 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2403397 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2404081 INFO (jetty-launcher-2861-thread-1) [n:127.0.0.1:53918_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node1/.
[junit4] 2> 2404086 INFO (jetty-launcher-2861-thread-2) [n:127.0.0.1:53919_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/.
[junit4] 2> 2405587 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2405591 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53916/solr ready
[junit4] 2> 2405619 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&maxShardsPerNode=2&name=dv_coll&action=CREATE&numShards=4&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2405623 INFO (OverseerThreadFactory-8238-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.c.CreateCollectionCmd Create collection dv_coll
[junit4] 2> 2405745 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=dv_coll_shard3_replica1&action=CREATE&numShards=4&collection=dv_coll&shard=shard3&wt=javabin&version=2
[junit4] 2> 2405745 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=dv_coll_shard4_replica1&action=CREATE&numShards=4&collection=dv_coll&shard=shard4&wt=javabin&version=2
[junit4] 2> 2405745 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=dv_coll_shard1_replica1&action=CREATE&numShards=4&collection=dv_coll&shard=shard1&wt=javabin&version=2
[junit4] 2> 2405749 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=dv_coll_shard2_replica1&action=CREATE&numShards=4&collection=dv_coll&shard=shard2&wt=javabin&version=2
[junit4] 2> 2405868 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2405868 INFO (zkCallback-2872-thread-3-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2406789 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2406790 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2406791 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2406796 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2406801 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.s.IndexSchema [dv_coll_shard2_replica1] Schema name=minimal
[junit4] 2> 2406806 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2406808 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.s.IndexSchema [dv_coll_shard1_replica1] Schema name=minimal
[junit4] 2> 2406811 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2406813 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard2_replica1' using configuration from collection dv_coll
[junit4] 2> 2406813 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.SolrCore [[dv_coll_shard2_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node1/dv_coll_shard2_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node1/./dv_coll_shard2_replica1/data/]
[junit4] 2> 2406815 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.s.IndexSchema [dv_coll_shard3_replica1] Schema name=minimal
[junit4] 2> 2406816 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.s.IndexSchema [dv_coll_shard4_replica1] Schema name=minimal
[junit4] 2> 2406817 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2406818 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2406819 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard1_replica1' using configuration from collection dv_coll
[junit4] 2> 2406819 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard3_replica1' using configuration from collection dv_coll
[junit4] 2> 2406819 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [[dv_coll_shard3_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/dv_coll_shard3_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/./dv_coll_shard3_replica1/data/]
[junit4] 2> 2406819 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.SolrCore [[dv_coll_shard1_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/dv_coll_shard1_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/./dv_coll_shard1_replica1/data/]
[junit4] 2> 2406821 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard4_replica1' using configuration from collection dv_coll
[junit4] 2> 2406821 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.SolrCore [[dv_coll_shard4_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node1/dv_coll_shard4_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node1/./dv_coll_shard4_replica1/data/]
[junit4] 2> 2406862 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2406862 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2406865 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2406865 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2406875 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6e42e2bb[dv_coll_shard2_replica1] main]
[junit4] 2> 2406876 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2406876 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2406879 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2406879 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2406880 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2406880 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2406881 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2406881 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2406883 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2406883 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2406883 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2406884 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2406884 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
[junit4] 2> 2406885 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2406885 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2406885 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 2406886 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2406886 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557542115482796032
[junit4] 2> 2406886 INFO (searcherExecutor-8243-thread-1-processing-n:127.0.0.1:53918_solr x:dv_coll_shard2_replica1 s:shard2 c:dv_coll) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.SolrCore [dv_coll_shard2_replica1] Registered new searcher Searcher@6e42e2bb[dv_coll_shard2_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2406887 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6517068e[dv_coll_shard3_replica1] main]
[junit4] 2> 2406890 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2406890 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@67d8b358[dv_coll_shard4_replica1] main]
[junit4] 2> 2406893 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2406894 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2406895 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2406895 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
[junit4] 2> 2406898 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 2406898 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2406898 INFO (searcherExecutor-8246-thread-1-processing-n:127.0.0.1:53918_solr x:dv_coll_shard4_replica1 s:shard4 c:dv_coll) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.SolrCore [dv_coll_shard4_replica1] Registered new searcher Searcher@67d8b358[dv_coll_shard4_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2406899 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557542115496427520
[junit4] 2> 2406903 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
[junit4] 2> 2406905 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 2406906 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2406906 INFO (searcherExecutor-8245-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard3_replica1 s:shard3 c:dv_coll) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] Registered new searcher Searcher@6517068e[dv_coll_shard3_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2406906 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557542115503767552
[junit4] 2> 2406913 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2406913 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2406913 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53918/solr/dv_coll_shard4_replica1/
[junit4] 2> 2406913 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2406913 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:53918/solr/dv_coll_shard4_replica1/ has no replicas
[junit4] 2> 2406917 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2406918 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2406918 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53918/solr/dv_coll_shard2_replica1/
[junit4] 2> 2406918 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2406919 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:53918/solr/dv_coll_shard2_replica1/ has no replicas
[junit4] 2> 2406923 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2406923 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2406923 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/
[junit4] 2> 2406923 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2406923 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/ has no replicas
[junit4] 2> 2406924 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53918/solr/dv_coll_shard4_replica1/ shard4
[junit4] 2> 2406930 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53918/solr/dv_coll_shard2_replica1/ shard2
[junit4] 2> 2406930 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/ shard3
[junit4] 2> 2406936 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5c9c30de[dv_coll_shard1_replica1] main]
[junit4] 2> 2406939 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2406940 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2406940 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
[junit4] 2> 2406942 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 2406942 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2406942 INFO (searcherExecutor-8244-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard1_replica1 s:shard1 c:dv_coll) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.SolrCore [dv_coll_shard1_replica1] Registered new searcher Searcher@5c9c30de[dv_coll_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2406942 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557542115541516288
[junit4] 2> 2406955 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2406955 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2406955 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/
[junit4] 2> 2406955 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2406955 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/ has no replicas
[junit4] 2> 2406963 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/ shard1
[junit4] 2> 2407075 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2407076 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2407081 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2407087 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=dv_coll_shard4_replica1&action=CREATE&numShards=4&collection=dv_coll&shard=shard4&wt=javabin&version=2} status=0 QTime=1342
[junit4] 2> 2407097 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2407098 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2407108 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2407103 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=dv_coll_shard2_replica1&action=CREATE&numShards=4&collection=dv_coll&shard=shard2&wt=javabin&version=2} status=0 QTime=1355
[junit4] 2> 2407115 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=dv_coll_shard3_replica1&action=CREATE&numShards=4&collection=dv_coll&shard=shard3&wt=javabin&version=2} status=0 QTime=1370
[junit4] 2> 2407129 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=dv_coll_shard1_replica1&action=CREATE&numShards=4&collection=dv_coll&shard=shard1&wt=javabin&version=2} status=0 QTime=1383
[junit4] 2> 2407138 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2407229 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2407229 INFO (zkCallback-2872-thread-3-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2407638 INFO (OverseerCollectionConfigSetProcessor-97346381959790598-127.0.0.1:53919_solr-n_0000000000) [n:127.0.0.1:53919_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2408140 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&maxShardsPerNode=2&name=dv_coll&action=CREATE&numShards=4&wt=javabin&version=2} status=0 QTime=2521
[junit4] 2> 2408189 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.SchemaManager managed schema loaded . version : 0
[junit4] 2> 2408199 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.IndexSchema [dv_coll_shard3_replica1] Schema name=minimal
[junit4] 2> 2408201 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2408214 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.ZkController Persisted config data to node /configs/conf1/managed-schema
[junit4] 2> 2408215 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
[junit4] 2> 2408215 INFO (zkCallback-2871-thread-2-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
[junit4] 2> 2408215 INFO (zkCallback-2872-thread-3-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
[junit4] 2> 2408218 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
[junit4] 2> 2408218 INFO (zkCallback-2871-thread-2-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
[junit4] 2> 2408220 INFO (zkCallback-2871-thread-2-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.IndexSchema [dv_coll_shard2_replica1] Schema name=minimal
[junit4] 2> 2408223 INFO (zkCallback-2871-thread-2-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2408224 INFO (zkCallback-2871-thread-2-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 5 ms
[junit4] 2> 2408224 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
[junit4] 2> 2408226 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.IndexSchema [dv_coll_shard1_replica1] Schema name=minimal
[junit4] 2> 2408229 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
[junit4] 2> 2408229 INFO (Thread-3509) [n:127.0.0.1:53919_solr ] o.a.s.c.SolrCore config update listener called for core dv_coll_shard3_replica1
[junit4] 2> 2408230 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2408231 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 6 ms
[junit4] 2> 2408231 INFO (Thread-3510) [n:127.0.0.1:53918_solr ] o.a.s.c.SolrCore config update listener called for core dv_coll_shard2_replica1
[junit4] 2> 2408232 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.IndexSchema [dv_coll_shard4_replica1] Schema name=minimal
[junit4] 2> 2408236 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2408236 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 6 ms
[junit4] 2> 2408237 INFO (Thread-3509) [n:127.0.0.1:53919_solr ] o.a.s.c.SolrCore core reload dv_coll_shard3_replica1
[junit4] 2> 2408238 INFO (Thread-3510) [n:127.0.0.1:53918_solr ] o.a.s.c.SolrCore config update listener called for core dv_coll_shard4_replica1
[junit4] 2> 2408243 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2408250 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.IndexSchema [dv_coll_shard3_replica1] Schema name=minimal
[junit4] 2> 2408254 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2408255 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.CoreContainer Reloading SolrCore 'dv_coll_shard3_replica1' using configuration from collection dv_coll
[junit4] 2> 2408256 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [[dv_coll_shard3_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/dv_coll_shard3_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/./dv_coll_shard3_replica1/data/]
[junit4] 2> 2408278 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2408278 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2408278 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1f08b308[dv_coll_shard3_replica1] main]
[junit4] 2> 2408280 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2408281 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2408281 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
[junit4] 2> 2408282 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.ZkIndexSchemaReader Current schema version 1 is already the latest
[junit4] 2> 2408282 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2408282 INFO (searcherExecutor-8263-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard3_replica1 s:shard3 c:dv_coll r:core_node2) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] Registered new searcher Searcher@1f08b308[dv_coll_shard3_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408283 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557542116947656704
[junit4] 2> 2408291 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 2408291 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7765a366[dv_coll_shard3_replica1] main]
[junit4] 2> 2408292 INFO (searcherExecutor-8263-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard3_replica1 s:shard3 c:dv_coll r:core_node2) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] Registered new searcher Searcher@7765a366[dv_coll_shard3_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408292 INFO (zkCallback-2872-thread-3-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
[junit4] 2> 2408296 INFO (zkCallback-2872-thread-3-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.IndexSchema [dv_coll_shard3_replica1] Schema name=minimal
[junit4] 2> 2408323 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.ManagedIndexSchema Waiting up to 599 secs for 3 replicas to apply schema update version 1 for collection dv_coll
[junit4] 2> 2408323 INFO (Thread-3509) [n:127.0.0.1:53919_solr ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2408323 INFO (zkCallback-2872-thread-3-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2408323 INFO (zkCallback-2872-thread-3-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 30 ms
[junit4] 2> 2408329 INFO (qtp1276199975-16307) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.c.S.Request [dv_coll_shard1_replica1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=1&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2408330 INFO (qtp1113000480-16296) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.c.S.Request [dv_coll_shard2_replica1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=1&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2408331 INFO (qtp1113000480-16296) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.c.S.Request [dv_coll_shard4_replica1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=1&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2408332 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.ManagedIndexSchema Took 40.0ms for 3 replicas to apply schema update version 1 for collection dv_coll
[junit4] 2> 2408333 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.S.Request [dv_coll_shard3_replica1] webapp=/solr path=/schema params={wt=javabin&version=2} status=0 QTime=147
[junit4] 2> 2408333 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@27b1385a
[junit4] 2> 2408333 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.dv_coll.shard3.replica1
[junit4] 2> 2408334 INFO (Thread-3509) [n:127.0.0.1:53919_solr ] o.a.s.s.IndexSchema [dv_coll_shard3_replica1] Schema name=minimal
[junit4] 2> 2408348 INFO (Thread-3509) [n:127.0.0.1:53919_solr ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2408352 INFO (Thread-3509) [n:127.0.0.1:53919_solr ] o.a.s.c.CoreContainer Reloading SolrCore 'dv_coll_shard3_replica1' using configuration from collection dv_coll
[junit4] 2> 2408357 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [[dv_coll_shard3_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/dv_coll_shard3_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/./dv_coll_shard3_replica1/data/]
[junit4] 2> 2408366 INFO (TEST-DocValuesNotIndexedTest.testGroupingSorting-seed#[36B7C7B063946B0B]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testGroupingSorting
[junit4] 2> 2408375 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica1] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:53918/solr/dv_coll_shard2_replica1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557542117040979968)} 0 3
[junit4] 2> 2408377 INFO (qtp1113000480-16297) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica1] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:53918/solr/dv_coll_shard2_replica1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557542117039931392)} 0 5
[junit4] 2> 2408377 INFO (qtp1276199975-16307) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:53918/solr/dv_coll_shard2_replica1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557542117039931392)} 0 6
[junit4] 2> 2408378 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1557542117037834240)} 0 10
[junit4] 2> 2408379 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2408379 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2408379 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@453142b6[dv_coll_shard3_replica1] main]
[junit4] 2> 2408380 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408380 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2408381 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408381 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1d70f30[dv_coll_shard1_replica1] main]
[junit4] 2> 2408382 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2408382 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408382 INFO (searcherExecutor-8244-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard1_replica1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.c.SolrCore [dv_coll_shard1_replica1] Registered new searcher Searcher@1d70f30[dv_coll_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408382 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6e7cced2[dv_coll_shard2_replica1] main]
[junit4] 2> 2408382 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 2408382 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408382 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2408382 INFO (searcherExecutor-8243-thread-1-processing-n:127.0.0.1:53918_solr x:dv_coll_shard2_replica1 s:shard2 c:dv_coll r:core_node4) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.c.SolrCore [dv_coll_shard2_replica1] Registered new searcher Searcher@6e7cced2[dv_coll_shard2_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408382 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408383 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 2
[junit4] 2> 2408383 INFO (qtp1113000480-16296) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408383 INFO (qtp1113000480-16296) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2408384 INFO (qtp1113000480-16296) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6aad5317[dv_coll_shard4_replica1] main]
[junit4] 2> 2408384 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2408384 INFO (qtp1113000480-16296) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408384 INFO (searcherExecutor-8246-thread-1-processing-n:127.0.0.1:53918_solr x:dv_coll_shard4_replica1 s:shard4 c:dv_coll r:core_node1) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.c.SolrCore [dv_coll_shard4_replica1] Registered new searcher Searcher@6aad5317[dv_coll_shard4_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408384 INFO (qtp1113000480-16296) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 2408385 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2408385 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
[junit4] 2> 2408386 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.ZkIndexSchemaReader Current schema version 1 is already the latest
[junit4] 2> 2408386 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2408386 INFO (searcherExecutor-8268-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard3_replica1 s:shard3 c:dv_coll r:core_node2) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] Registered new searcher Searcher@453142b6[dv_coll_shard3_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408386 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408387 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 4
[junit4] 2> 2408387 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 8
[junit4] 2> 2408393 INFO (qtp1113000480-16299) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&wt=javabin&version=2}{add=[4 (1557542117058805760)]} 0 4
[junit4] 2> 2408395 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&wt=javabin&version=2}{add=[1 (1557542117059854336)]} 0 4
[junit4] 2> 2408451 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 2408453 INFO (qtp1276199975-16301) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&wt=javabin&version=2}{add=[2 (1557542117059854336), 3 (1557542117063000064)]} 0 64
[junit4] 2> 2408455 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408455 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408455 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@2f684653
[junit4] 2> 2408455 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408455 INFO (qtp1113000480-16295) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408455 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@4b0c9b2e
[junit4] 2> 2408455 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@204e25a6
[junit4] 2> 2408455 INFO (qtp1113000480-16295) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2408459 INFO (qtp1113000480-16295) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408459 INFO (qtp1113000480-16295) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 4
[junit4] 2> 2408466 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5451e0e3[dv_coll_shard3_replica1] main]
[junit4] 2> 2408467 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore config update listener called for core dv_coll_shard1_replica1
[junit4] 2> 2408467 INFO (searcherExecutor-8268-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard3_replica1 s:shard3 c:dv_coll r:core_node2) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] Registered new searcher Searcher@5451e0e3[dv_coll_shard3_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):C1)))}
[junit4] 2> 2408498 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@43c56d3a[dv_coll_shard2_replica1] main]
[junit4] 2> 2408499 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408499 INFO (searcherExecutor-8243-thread-1-processing-n:127.0.0.1:53918_solr x:dv_coll_shard2_replica1 s:shard2 c:dv_coll r:core_node4) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.c.SolrCore [dv_coll_shard2_replica1] Registered new searcher Searcher@43c56d3a[dv_coll_shard2_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):C1)))}
[junit4] 2> 2408499 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 44
[junit4] 2> 2408515 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4c4bd8da[dv_coll_shard1_replica1] main]
[junit4] 2> 2408515 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408515 INFO (searcherExecutor-8244-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard1_replica1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.c.SolrCore [dv_coll_shard1_replica1] Registered new searcher Searcher@4c4bd8da[dv_coll_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):C1)))}
[junit4] 2> 2408516 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 61
[junit4] 2> 2408517 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3f2ccaac[dv_coll_shard3_replica1] main]
[junit4] 2> 2408517 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408517 INFO (searcherExecutor-8263-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard3_replica1 s:shard3 c:dv_coll r:core_node2) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] Registered new searcher Searcher@3f2ccaac[dv_coll_shard3_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):C1) Uninverting(_1(7.0.0):C1)))}
[junit4] 2> 2408517 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 62
[junit4] 2> 2408517 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@5f5b6715
[junit4] 2> 2408518 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.dv_coll.shard3.replica1
[junit4] 2> 2408518 INFO (qtp1276199975-16307) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&commit=true&wt=javabin&version=2}{commit=} 0 64
[junit4] 2> 2408522 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.c.S.Request [dv_coll_shard2_replica1] webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=dv_coll:3&fl=id&shards.purpose=4&start=0&fsv=true&sort=intGSF+asc,id+asc&shard.url=http://127.0.0.1:53918/solr/dv_coll_shard2_replica1/&rows=100&version=2&q=*:*&NOW=1485387913890&isShard=true&wt=javabin} hits=1 status=0 QTime=0
[junit4] 2> 2408523 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.c.S.Request [dv_coll_shard4_replica1] webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=dv_coll:3&fl=id&shards.purpose=4&start=0&fsv=true&sort=intGSF+asc,id+asc&shard.url=http://127.0.0.1:53918/solr/dv_coll_shard4_replica1/&rows=100&version=2&q=*:*&NOW=1485387913890&isShard=true&wt=javabin} hits=0 status=0 QTime=0
[junit4] 2> 2408523 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.S.Request [dv_coll_shard3_replica1] webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=dv_coll:3&fl=id&shards.purpose=4&start=0&fsv=true&sort=intGSF+asc,id+asc&shard.url=http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/&rows=100&version=2&q=*:*&NOW=1485387913890&isShard=true&wt=javabin} hits=1 status=0 QTime=0
[junit4] 2> 2408524 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.c.S.Request [dv_coll_shard1_replica1] webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=dv_coll:3&fl=id&shards.purpose=4&start=0&fsv=true&sort=intGSF+asc,id+asc&shard.url=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&rows=100&version=2&q=*:*&NOW=1485387913890&isShard=true&wt=javabin} hits=1 status=0 QTime=0
[junit4] 2> 2408526 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.c.S.Request [dv_coll_shard2_replica1] webapp=/solr path=/select params={q=*:*&df=text&distrib=false&_stateVer_=dv_coll:3&shards.purpose=64&NOW=1485387913890&ids=4&isShard=true&shard.url=http://127.0.0.1:53918/solr/dv_coll_shard2_replica1/&rows=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2408526 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.c.S.Request [dv_coll_shard1_replica1] webapp=/solr path=/select params={q=*:*&df=text&distrib=false&_stateVer_=dv_coll:3&shards.purpose=64&NOW=1485387913890&ids=1&isShard=true&shard.url=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&rows=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2408526 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.S.Request [dv_coll_shard3_replica1] webapp=/solr path=/select params={q=*:*&df=text&distrib=false&_stateVer_=dv_coll:3&shards.purpose=64&NOW=1485387913890&ids=2&isShard=true&shard.url=http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/&rows=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2408527 INFO (qtp1113000480-16297) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.c.S.Request [dv_coll_shard4_replica1] webapp=/solr path=/select params={q=*:*&_stateVer_=dv_coll:3&sort=intGSF+asc,id+asc&rows=100&wt=javabin&version=2} hits=3 status=0 QTime=7
[junit4] 2> 2408528 INFO (TEST-DocValuesNotIndexedTest.testGroupingSorting-seed#[36B7C7B063946B0B]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testGroupingSorting
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=DocValuesNotIndexedTest -Dtests.method=testGroupingSorting -Dtests.seed=36B7C7B063946B0B -Dtests.slow=true -Dtests.locale=en-US -Dtests.timezone=Pacific/Enderbury -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] FAILURE 0.20s J1 | DocValuesNotIndexedTest.testGroupingSorting <<<
[junit4] > Throwable #1: java.lang.AssertionError: Should have exactly 4 documents returned expected:<4> but was:<3>
[junit4] > at __randomizedtesting.SeedInfo.seed([36B7C7B063946B0B:288FCFB81F3FD18B]:0)
[junit4] > at org.apache.solr.cloud.DocValuesNotIndexedTest.checkSortOrder(DocValuesNotIndexedTest.java:259)
[junit4] > at org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingSorting(DocValuesNotIndexedTest.java:244)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2408558 INFO (TEST-DocValuesNotIndexedTest.testGroupingDocAbsent-seed#[36B7C7B063946B0B]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testGroupingDocAbsent
[junit4] 2> 2408562 INFO (qtp1113000480-16295) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3f5d8bd1[dv_coll_shard2_replica1] realtime]
[junit4] 2> 2408562 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1c36232c[dv_coll_shard1_replica1] realtime]
[junit4] 2> 2408563 INFO (qtp1113000480-16295) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica1] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:53918/solr/dv_coll_shard4_replica1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557542117237063680)} 0 3
[junit4] 2> 2408563 INFO (qtp1276199975-16301) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@454f76a7[dv_coll_shard3_replica1] realtime]
[junit4] 2> 2408563 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:53918/solr/dv_coll_shard4_replica1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557542117237063680)} 0 3
[junit4] 2> 2408563 INFO (qtp1276199975-16301) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica1] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:53918/solr/dv_coll_shard4_replica1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557542117237063680)} 0 3
[junit4] 2> 2408563 INFO (qtp1113000480-16299) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1557542117237063680)} 0 5
[junit4] 2> 2408566 INFO (qtp1276199975-16307) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408566 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408566 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408566 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@2f684653
[junit4] 2> 2408566 INFO (qtp1276199975-16307) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@204e25a6
[junit4] 2> 2408566 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@4b0c9b2e
[junit4] 2> 2408566 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408566 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2408566 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@14cba9f9[dv_coll_shard2_replica1] main]
[junit4] 2> 2408567 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408567 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
[junit4] 2> 2408567 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408567 INFO (searcherExecutor-8243-thread-1-processing-n:127.0.0.1:53918_solr x:dv_coll_shard2_replica1 s:shard2 c:dv_coll r:core_node4) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.c.SolrCore [dv_coll_shard2_replica1] Registered new searcher Searcher@14cba9f9[dv_coll_shard2_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408567 INFO (qtp1276199975-16307) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@53bc3164[dv_coll_shard1_replica1] main]
[junit4] 2> 2408568 INFO (qtp1276199975-16307) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408568 INFO (searcherExecutor-8244-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard1_replica1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.c.SolrCore [dv_coll_shard1_replica1] Registered new searcher Searcher@53bc3164[dv_coll_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408568 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 2
[junit4] 2> 2408568 INFO (qtp1276199975-16307) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 3
[junit4] 2> 2408569 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@21174ffa[dv_coll_shard3_replica1] main]
[junit4] 2> 2408569 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408569 INFO (searcherExecutor-8268-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard3_replica1 s:shard3 c:dv_coll r:core_node2) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] Registered new searcher Searcher@21174ffa[dv_coll_shard3_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408569 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 3
[junit4] 2> 2408570 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 4
[junit4] 2> 2408573 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&wt=javabin&version=2}{add=[1 (1557542117249646592)]} 0 2
[junit4] 2> 2408574 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&wt=javabin&version=2}{add=[2 (1557542117250695168), 3 (1557542117252792320)]} 0 2
[junit4] 2> 2408574 INFO (qtp1113000480-16296) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&wt=javabin&version=2}{add=[4 (1557542117249646592)]} 0 3
[junit4] 2> 2408576 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408576 INFO (qtp1276199975-16301) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408577 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2408577 INFO (qtp1276199975-16301) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@204e25a6
[junit4] 2> 2408577 INFO (qtp1113000480-16297) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408577 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408577 INFO (q
[...truncated too long message...]
e_nodes]
[junit4] 2> 2410675 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.c.Overseer Overseer (id=97346381959790598-127.0.0.1:53919_solr-n_0000000000) closing
[junit4] 2> 2410675 INFO (jetty-launcher-2861-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 2410675 INFO (OverseerStateUpdate-97346381959790598-127.0.0.1:53919_solr-n_0000000000) [n:127.0.0.1:53919_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:53919_solr
[junit4] 2> 2410675 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 2410676 INFO (jetty-launcher-2861-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6417bf1d{/solr,null,UNAVAILABLE}
[junit4] 2> 2412178 WARN (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 2412179 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 2412179 INFO (jetty-launcher-2861-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@574b78af{/solr,null,UNAVAILABLE}
[junit4] 2> 2412201 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53916 53916
[junit4] 2> 2416494 INFO (Thread-3490) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53916 53916
[junit4] 2> 2419026 WARN (Thread-3490) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/configs/conf1/managed-schema
[junit4] 2> 4 /solr/configs/conf1
[junit4] 2> 3 /solr/aliases.json
[junit4] 2> 3 /solr/clusterprops.json
[junit4] 2> 2 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/clusterstate.json
[junit4] 2> 2 /solr/configs/conf1/managed-schema
[junit4] 2> 2 /solr/collections/dv_coll/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 13 /solr/overseer/queue
[junit4] 2> 7 /solr/overseer/collection-queue-work
[junit4] 2> 3 /solr/live_nodes
[junit4] 2> 3 /solr/collections
[junit4] 2> 3 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> 2419027 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {_version_=FSTOrd50, id=PostingsFormat(name=LuceneFixedGap)}, docValues:{floatField=DocValuesFormat(name=Lucene70), doubleField=DocValuesFormat(name=Asserting), intFieldMulti=DocValuesFormat(name=Lucene70), boolFieldMulti=DocValuesFormat(name=Asserting), floatGSL=DocValuesFormat(name=Lucene70), longFieldMulti=DocValuesFormat(name=Memory), intField=DocValuesFormat(name=Asserting), boolField=DocValuesFormat(name=Lucene70), stringFieldMulti=DocValuesFormat(name=Lucene70), dateFieldMulti=DocValuesFormat(name=Memory), longGSF=DocValuesFormat(name=Lucene70), doubleFieldMulti=DocValuesFormat(name=Lucene70), stringGSL=DocValuesFormat(name=Memory), doubleGSF=DocValuesFormat(name=Asserting), doubleGSL=DocValuesFormat(name=Memory), stringGSF=DocValuesFormat(name=Asserting), intGSL=DocValuesFormat(name=Asserting), longField=DocValuesFormat(name=Lucene70), dateField=DocValuesFormat(name=Lucene70), dateGSF=DocValuesFormat(name=Lucene70), boolGSF=DocValuesFormat(name=Lucene70), longGSL=DocValuesFormat(name=Lucene70), boolGSL=DocValuesFormat(name=Lucene70), floatFieldMulti=DocValuesFormat(name=Asserting), floatGSF=DocValuesFormat(name=Lucene70), dateGSL=DocValuesFormat(name=Lucene70), intGSF=DocValuesFormat(name=Memory), stringField=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=994, maxMBSortInHeap=5.7251959539859705, sim=RandomSimilarity(queryNorm=true): {}, locale=en-US, timezone=Pacific/Enderbury
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=244125368,total=490733568
[junit4] 2> NOTE: All tests run in this JVM: [TestSolrCloudWithKerberosAlt, DocValuesMissingTest, SolrIndexMetricsTest, TestComplexPhraseQParserPlugin, VMParamsZkACLAndCredentialsProvidersTest, SolrCLIZkUtilsTest, DistanceUnitsTest, TestScoreJoinQPScore, TestSchemaSimilarityResource, TestBadConfig, SolrMetricsIntegrationTest, TestCloudSchemaless, DeleteInactiveReplicaTest, TestImplicitCoreProperties, TestBulkSchemaAPI, TestTrackingShardHandlerFactory, UpdateParamsTest, AsyncCallRequestStatusResponseTest, SuggesterTest, TestDynamicFieldCollectionResource, TestInitParams, BadIndexSchemaTest, BasicFunctionalityTest, TestConfigSetImmutable, LukeRequestHandlerTest, SolrMetricManagerTest, TestDFRSimilarityFactory, ShardSplitTest, HdfsBasicDistributedZkTest, TestJoin, TestHdfsBackupRestoreCore, EchoParamsTest, TestXmlQParserPlugin, StatsComponentTest, TestInitQParser, TestManagedSchemaThreadSafety, OverseerRolesTest, TestSchemaNameResource, DistributedFacetExistsSmallTest, DistributedIntervalFacetingTest, TestQueryUtils, URLClassifyProcessorTest, TestSegmentSorting, RankQueryTest, AnalyticsMergeStrategyTest, TestSolrConfigHandler, TestCrossCoreJoin, CdcrBootstrapTest, CollectionsAPIDistributedZkTest, TestFuzzyAnalyzedSuggestions, RegexBoostProcessorTest, DataDrivenBlockJoinTest, CdcrVersionReplicationTest, ReturnFieldsTest, TestConfig, TestClassicSimilarityFactory, TestLMJelinekMercerSimilarityFactory, FullHLLTest, UUIDFieldTest, HdfsSyncSliceTest, DistributedTermsComponentTest, TestTolerantUpdateProcessorCloud, HdfsNNFailoverTest, ForceLeaderTest, TestComponentsName, TestJmxMonitoredMap, TestSolrXml, MissingSegmentRecoveryTest, CoreAdminCreateDiscoverTest, AtomicUpdatesTest, TestLMDirichletSimilarityFactory, SpellCheckComponentTest, TestManagedStopFilterFactory, TestDistributedMissingSort, LeaderInitiatedRecoveryOnShardRestartTest, TestBlendedInfixSuggestions, RequestHandlersTest, DefaultValueUpdateProcessorTest, DistributedSuggestComponentTest, TestRangeQuery, TestDistribIDF, JsonLoaderTest, TestReversedWildcardFilterFactory, TestExpandComponent, SignatureUpdateProcessorFactoryTest, TemplateUpdateProcessorTest, DeleteNodeTest, HighlighterConfigTest, TestFreeTextSuggestions, TestRTimerTree, TestStandardQParsers, BlockCacheTest, CollectionTooManyReplicasTest, SpatialRPTFieldTypeTest, AnalysisErrorHandlingTest, TestBackupRepositoryFactory, TestZkChroot, HdfsTlogReplayBufferedWhileIndexingTest, ReplaceNodeTest, ResponseLogComponentTest, ShowFileRequestHandlerTest, TestSolrCoreParser, TestSolrQueryParserResource, TestSearchPerf, HLLUtilTest, RestartWhileUpdatingTest, TestSQLHandler, TestLuceneMatchVersion, QueryParsingTest, BasicAuthIntegrationTest, TermVectorComponentTest, TestCloudDeleteByQuery, DistanceFunctionTest, MetricUtilsTest, TestSolrCoreProperties, TestFoldingMultitermQuery, TestCopyFieldCollectionResource, TestExceedMaxTermLength, TestTrieFacet, TestSolrDeletionPolicy2, TestBlobHandler, TestCloudManagedSchema, TestTrie, TestSolr4Spatial, TestSubQueryTransformerDistrib, SolrIndexSplitterTest, TestSmileRequest, CdcrReplicationHandlerTest, PreAnalyzedFieldTest, SolrInfoMBeanTest, TestOnReconnectListenerSupport, TestEmbeddedSolrServerConstructors, SolrCoreCheckLockOnStartupTest, DeleteShardTest, DistributedFacetPivotSmallTest, TestFieldCollectionResource, TestReloadAndDeleteDocs, SuggestComponentTest, TestIndexSearcher, TestRecovery, TestUtils, ResourceLoaderTest, TestFieldCacheSanityChecker, TestRandomRequestDistribution, TestSystemIdResolver, ZkStateWriterTest, HdfsRecoveryZkTest, SpellCheckCollatorTest, TestLeaderElectionWithEmptyReplica, TestCryptoKeys, HighlighterMaxOffsetTest, SolrXmlInZkTest, WordBreakSolrSpellCheckerTest, LeaderElectionIntegrationTest, ClusterStateTest, TestCustomDocTransformer, TestFieldTypeCollectionResource, CloudMLTQParserTest, OverseerTaskQueueTest, CurrencyFieldOpenExchangeTest, TestLockTree, ReplicaListTransformerTest, HdfsLockFactoryTest, TestConfigSets, RAMDirectoryFactoryTest, TestSlowCompositeReaderWrapper, TestQuerySenderListener, SyncSliceTest, RecoveryZkTest, ClusterStateUpdateTest, TestRandomFaceting, ZkSolrClientTest, ZkCLITest, TermVectorComponentDistributedTest, TestRealTimeGet, TestStressVersions, SolrCoreTest, SolrCmdDistributorTest, TestFunctionQuery, TestLazyCores, SoftAutoCommitTest, CoreAdminHandlerTest, NoCacheHeaderTest, TestPseudoReturnFields, FieldMutatingUpdateProcessorTest, TestAtomicUpdateErrorCases, StatelessScriptUpdateProcessorFactoryTest, CacheHeaderTest, TestWriterPerf, FileBasedSpellCheckerTest, PathHierarchyTokenizerFactoryTest, FieldAnalysisRequestHandlerTest, IndexSchemaRuntimeFieldTest, IndexSchemaTest, UpdateRequestProcessorFactoryTest, TestPartialUpdateDeduplication, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, TestSolrIndexConfig, CopyFieldTest, SolrIndexConfigTest, BadComponentTest, TestStressRecovery, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestHighlightDedupGrouping, TestSimpleTrackingShardHandler, TestJettySolrRunner, ConnectionReuseTest, ActionThrottleTest, AliasIntegrationTest, CdcrReplicationDistributedZkTest, CollectionReloadTest, CollectionsAPISolrJTest, DeleteStatusTest, DistribJoinFromCollectionTest, DistributedVersionInfoTest, DocValuesNotIndexedTest]
[junit4] Completed [487/680 (1!)] on J1 in 16.13s, 4 tests, 1 failure <<< FAILURES!
[...truncated 63069 lines...]
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3797
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3797/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC
1 tests failed.
FAILED: org.apache.solr.cloud.PeerSyncReplicationTest.test
Error Message:
timeout waiting to see all nodes active
Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
at __randomizedtesting.SeedInfo.seed([A06A4D09107E4409:283E72D3BE8229F1]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 11457 lines...]
[junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/init-core-data-001
[junit4] 2> 1019284 INFO (SUITE-PeerSyncReplicationTest-seed#[A06A4D09107E4409]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields
[junit4] 2> 1019288 INFO (SUITE-PeerSyncReplicationTest-seed#[A06A4D09107E4409]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 1019289 INFO (SUITE-PeerSyncReplicationTest-seed#[A06A4D09107E4409]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 1019290 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1019291 INFO (Thread-1655) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1019291 INFO (Thread-1655) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1019395 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ZkTestServer start zk server on port:60300
[junit4] 2> 1019425 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1019432 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 1019436 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1019440 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1019443 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1019446 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1019449 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1019453 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1019456 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1019459 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1019463 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1023112 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/control-001/cores/collection1
[junit4] 2> 1023115 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1023117 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4d355d8e{/,null,AVAILABLE}
[junit4] 2> 1023119 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5d1fcf0a{HTTP/1.1,[http/1.1]}{127.0.0.1:60303}
[junit4] 2> 1023119 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.Server Started @1028156ms
[junit4] 2> 1023119 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/tempDir-001/control/data, hostContext=/, hostPort=60303, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/control-001/cores}
[junit4] 2> 1023119 ERROR (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1023119 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1023120 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1023120 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1023120 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-25T15:11:30.446Z
[junit4] 2> 1023124 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1023124 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/control-001/solr.xml
[junit4] 2> 1023138 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60300/solr
[junit4] 2> 1023184 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60303_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60303_
[junit4] 2> 1023186 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60303_ ] o.a.s.c.Overseer Overseer (id=97344362028793860-127.0.0.1:60303_-n_0000000000) starting
[junit4] 2> 1023211 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60303_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60303_
[junit4] 2> 1023214 INFO (OverseerStateUpdate-97344362028793860-127.0.0.1:60303_-n_0000000000) [n:127.0.0.1:60303_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1023358 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60303_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/control-001/cores
[junit4] 2> 1023358 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60303_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1023363 INFO (OverseerStateUpdate-97344362028793860-127.0.0.1:60303_-n_0000000000) [n:127.0.0.1:60303_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1024386 WARN (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 1024387 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1024399 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1024486 WARN (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1024488 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1024519 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 1024519 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/control-001/cores/collection1/data/]
[junit4] 2> 1024519 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d0b4029
[junit4] 2> 1024525 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.35301399807770073]
[junit4] 2> 1024536 WARN (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1024552 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1024552 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1024554 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1024554 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1024554 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=76.486328125, floorSegmentMB=0.50390625, forceMergeDeletesPctAllowed=17.875198882979568, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7157930179055988
[junit4] 2> 1024555 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@c3aa56[collection1] main]
[junit4] 2> 1024557 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1024558 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1024558 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1024559 INFO (searcherExecutor-3457-thread-1-processing-n:127.0.0.1:60303_ x:collection1 c:control_collection) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@c3aa56[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1024559 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557509797980405760
[junit4] 2> 1024570 INFO (coreZkRegister-3449-thread-1-processing-n:127.0.0.1:60303_ x:collection1 c:control_collection) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1024570 INFO (coreZkRegister-3449-thread-1-processing-n:127.0.0.1:60303_ x:collection1 c:control_collection) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1024570 INFO (coreZkRegister-3449-thread-1-processing-n:127.0.0.1:60303_ x:collection1 c:control_collection) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60303/collection1/
[junit4] 2> 1024570 INFO (coreZkRegister-3449-thread-1-processing-n:127.0.0.1:60303_ x:collection1 c:control_collection) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1024570 INFO (coreZkRegister-3449-thread-1-processing-n:127.0.0.1:60303_ x:collection1 c:control_collection) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:60303/collection1/ has no replicas
[junit4] 2> 1024577 INFO (coreZkRegister-3449-thread-1-processing-n:127.0.0.1:60303_ x:collection1 c:control_collection) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60303/collection1/ shard1
[junit4] 2> 1024712 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1024715 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60300/solr ready
[junit4] 2> 1024715 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1024715 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
[junit4] 2> 1024737 INFO (coreZkRegister-3449-thread-1-processing-n:127.0.0.1:60303_ x:collection1 c:control_collection) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1025417 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-1-001/cores/collection1
[junit4] 2> 1025419 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-1-001
[junit4] 2> 1025420 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1025421 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@581b9b23{/,null,AVAILABLE}
[junit4] 2> 1025422 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1787576a{HTTP/1.1,[http/1.1]}{127.0.0.1:60309}
[junit4] 2> 1025422 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.Server Started @1030459ms
[junit4] 2> 1025422 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=60309, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-1-001/cores}
[junit4] 2> 1025422 ERROR (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1025423 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1025423 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1025423 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1025423 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-25T15:11:32.749Z
[junit4] 2> 1025428 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1025428 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-1-001/solr.xml
[junit4] 2> 1025454 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60300/solr
[junit4] 2> 1025474 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1025483 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60309_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60309_
[junit4] 2> 1025488 INFO (zkCallback-1484-thread-2-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1025488 INFO (zkCallback-1488-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1025490 INFO (zkCallback-1494-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1025577 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60309_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-1-001/cores
[junit4] 2> 1025577 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60309_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1025583 INFO (OverseerStateUpdate-97344362028793860-127.0.0.1:60303_-n_0000000000) [n:127.0.0.1:60303_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1025696 INFO (zkCallback-1494-thread-2-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1026719 WARN (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 1026720 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1026733 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1026824 WARN (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1026826 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1026846 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 1026846 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 1026846 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d0b4029
[junit4] 2> 1026851 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.35301399807770073]
[junit4] 2> 1026967 WARN (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1026988 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1026988 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1026989 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1026989 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1026989 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=76.486328125, floorSegmentMB=0.50390625, forceMergeDeletesPctAllowed=17.875198882979568, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7157930179055988
[junit4] 2> 1026990 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@67247e01[collection1] main]
[junit4] 2> 1026992 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1026993 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1026993 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1026994 INFO (searcherExecutor-3468-thread-1-processing-n:127.0.0.1:60309_ x:collection1 c:collection1) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@67247e01[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1026995 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557509800534736896
[junit4] 2> 1027005 INFO (coreZkRegister-3462-thread-1-processing-n:127.0.0.1:60309_ x:collection1 c:collection1) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1027005 INFO (coreZkRegister-3462-thread-1-processing-n:127.0.0.1:60309_ x:collection1 c:collection1) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1027005 INFO (coreZkRegister-3462-thread-1-processing-n:127.0.0.1:60309_ x:collection1 c:collection1) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60309/collection1/
[junit4] 2> 1027005 INFO (coreZkRegister-3462-thread-1-processing-n:127.0.0.1:60309_ x:collection1 c:collection1) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1027005 INFO (coreZkRegister-3462-thread-1-processing-n:127.0.0.1:60309_ x:collection1 c:collection1) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:60309/collection1/ has no replicas
[junit4] 2> 1027013 INFO (coreZkRegister-3462-thread-1-processing-n:127.0.0.1:60309_ x:collection1 c:collection1) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60309/collection1/ shard1
[junit4] 2> 1027120 INFO (zkCallback-1494-thread-2-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1027166 INFO (coreZkRegister-3462-thread-1-processing-n:127.0.0.1:60309_ x:collection1 c:collection1) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1027280 INFO (zkCallback-1494-thread-2-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1027774 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-2-001/cores/collection1
[junit4] 2> 1027776 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-2-001
[junit4] 2> 1027776 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1027780 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@54caacac{/,null,AVAILABLE}
[junit4] 2> 1027781 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@13423b1e{HTTP/1.1,[http/1.1]}{127.0.0.1:60313}
[junit4] 2> 1027781 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.Server Started @1032819ms
[junit4] 2> 1027781 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=60313, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-2-001/cores}
[junit4] 2> 1027781 ERROR (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1027784 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1027784 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1027785 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1027785 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-25T15:11:35.111Z
[junit4] 2> 1027789 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1027790 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-2-001/solr.xml
[junit4] 2> 1027806 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60300/solr
[junit4] 2> 1027832 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1027853 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60313_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60313_
[junit4] 2> 1027857 INFO (zkCallback-1494-thread-2-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1027857 INFO (zkCallback-1488-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1027857 INFO (zkCallback-1484-thread-2-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1027859 INFO (zkCallback-1500-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1027940 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60313_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-2-001/cores
[junit4] 2> 1027940 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60313_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1027969 INFO (OverseerStateUpdate-97344362028793860-127.0.0.1:60303_-n_0000000000) [n:127.0.0.1:60303_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1028096 INFO (zkCallback-1500-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1028096 INFO (zkCallback-1494-thread-2-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1028985 WARN (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 1028986 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1029001 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1029115 WARN (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1029117 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1029142 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 1029143 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 1029143 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d0b4029
[junit4] 2> 1029148 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.35301399807770073]
[junit4] 2> 1029159 WARN (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1029179 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1029179 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1029180 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1029180 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1029182 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=76.486328125, floorSegmentMB=0.50390625, forceMergeDeletesPctAllowed=17.875198882979568, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7157930179055988
[junit4] 2> 1029183 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@57163345[collection1] main]
[junit4] 2> 1029187 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1029188 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1029188 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1029190 INFO (searcherExecutor-3479-thread-1-processing-n:127.0.0.1:60313_ x:collection1 c:collection1) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@57163345[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1029191 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557509802837409792
[junit4] 2> 1029198 INFO (coreZkRegister-3473-thread-1-processing-n:127.0.0.1:60313_ x:collection1 c:collection1) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 1029198 INFO (updateExecutor-1497-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 1029199 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 1029199 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 1029199 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 1029199 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 1029199 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:60309/collection1/] and I am [http://127.0.0.1:60313/collection1/]
[junit4] 2> 1029202 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:60309]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:60313_&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 1029206 INFO (qtp1284525073-8013) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 1029207 INFO (qtp1284525073-8013) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 1029207 INFO (qtp1284525073-8013) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:60313_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:60313","node_name":"127.0.0.1:60313_","state":"down"}
[junit4] 2> 1029310 INFO (zkCallback-1500-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1029310 INFO (zkCallback-1494-thread-2-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1029616 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-3-001/cores/collection1
[junit4] 2> 1029618 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-3-001
[junit4] 2> 1029618 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1029620 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3dba342c{/,null,AVAILABLE}
[junit4] 2> 1029620 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@b1921c{HTTP/1.1,[http/1.1]}{127.0.0.1:60318}
[junit4] 2> 1029620 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.Server Started @1034658ms
[junit4] 2> 1029620 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=60318, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-3-001/cores}
[junit4] 2> 1029621 ERROR (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1029622 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1029622 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1029622 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1029622 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-25T15:11:36.948Z
[junit4] 2> 1029626 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1029626 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-3-001/solr.xml
[junit4] 2> 1029648 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60300/solr
[junit4] 2> 1029669 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60318_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1029678 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60318_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60318_
[junit4] 2> 1029682 INFO (zkCallback-1500-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1029683 INFO (zkCallback-1484-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1029683 INFO (zkCallback-1494-thread-2-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1029683 INFO (zkCallback-1488-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1029684 INFO (zkCallback-1507-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1029733 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60318_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-3-001/cores
[junit4] 2> 1029733 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60318_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1029737 INFO (OverseerStateUpdate-97344362028793860-127.0.0.1:60303_-n_0000000000) [n:127.0.0.1:60303_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1029850 INFO (zkCallback-1494-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1029850 INFO (zkCallback-1507-thread-2-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1029850 INFO (zkCallback-1500-thread-2-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1030210 INFO (qtp1284525073-8013) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:60313_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:60313","node_name":"127.0.0.1:60313_","state":"recovering"}
[junit4] 2> 1030210 INFO (qtp1284525073-8013) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 1030210 INFO (qtp1284525073-8013) [n:127.0.0.1:60309_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:60313_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1003
[junit4] 2> 1030759 WARN (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 1030759 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1030781 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1030895 WARN (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1030897 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1030921 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 1030921 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 1030922 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d0b4029
[junit4] 2> 1030928 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.35301399807770073]
[junit4] 2> 1030937 WARN (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1030953 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1030953 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1030954 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1030954 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1030955 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=76.486328125, floorSegmentMB=0.50390625, forceMergeDeletesPctAllowed=17.875198882979568, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7157930179055988
[junit4] 2> 1030956 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@17cc6ccc[collection1] main]
[junit4] 2> 1030958 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1030959 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1030959 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1030960 INFO (searcherExecutor-3490-thread-1-processing-n:127.0.0.1:60318_ x:collection1 c:collection1) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@17cc6ccc[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1030961 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557509804693389312
[junit4] 2> 1030967 INFO (coreZkRegister-3484-thread-1-processing-n:127.0.0.1:60318_ x:collection1 c:collection1) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 1030967 INFO (updateExecutor-1504-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 1030968 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 1030968 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 1030968 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 1030968 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 1030968 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:60309/collection1/] and I am [http://127.0.0.1:60318/collection1/]
[junit4] 2> 1030973 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:60309]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:60318_&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 1030975 INFO (qtp1284525073-8017) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 1030976 INFO (qtp1284525073-8017) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 1030976 INFO (qtp1284525073-8017) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:60318_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:60318","node_name":"127.0.0.1:60318_","state":"down"}
[junit4] 2> 1031079 INFO (zkCallback-1507-thread-2-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1031079 INFO (zkCallback-1500-thread-2-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1031079 INFO (zkCallback-1494-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1031091 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 1031091 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30 for each attempt
[junit4] 2> 1031091 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
[junit4] 2> 1031983 INFO (qtp1284525073-8017) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:60318_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:60318","node_name":"127.0.0.1:60318_","state":"recovering"}
[junit4] 2> 1031984 INFO (qtp1284525073-8017) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 1031984 INFO (qtp1284525073-8017) [n:127.0.0.1:60309_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:60318_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1008
[junit4] 2> 1037211 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:60309/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 1037211 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:60313 START replicas=[http://127.0.0.1:60309/collection1/] nUpdates=1000
[junit4] 2> 1037214 INFO (qtp1284525073-8014) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1037214 INFO (qtp1284525073-8014) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 1037220 INFO (zkCallback-1500-thread-2-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1037220 INFO (zkCallback-1507-thread-2-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1037220 INFO (zkCallback-1494-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1038988 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:60309/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 1038988 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:60318 START replicas=[http://127.0.0.1:60309/collection1/] nUpdates=1000
[junit4] 2> 1038990 INFO (qtp1284525073-8015) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1038990 INFO (qtp1284525073-8015) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1038991 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1038991 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 1038991 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1038991 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1038991 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1038991 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 1038991 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 1038992 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 1038992 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 1038996 INFO (zkCallback-1500-thread-2-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1038996 INFO (zkCallback-1494-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1038996 INFO (zkCallback-1507-thread-2-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1039128 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
[junit4] 2> 1039134 INFO (qtp1186743350-7973) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1039134 INFO (qtp1186743350-7973) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1039135 INFO (qtp1186743350-7973) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1039135 INFO (qtp1186743350-7973) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 1
[junit4] 2> 1039139 INFO (qtp1284525073-8018) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1039139 INFO (qtp1284525073-8018) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1039140 INFO (qtp1284525073-8018) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1039141 INFO (qtp1284525073-8018) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:60309/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 1039142 INFO (qtp24934752-8043) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1039142 INFO (qtp294505202-8078) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1039142 INFO (qtp24934752-8043) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1039142 INFO (qtp294505202-8078) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1039143 INFO (qtp24934752-8043) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1039143 INFO (qtp24934752-8043) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:60309/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
[junit4] 2> 1039143 INFO (qtp294505202-8078) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1039143 INFO (qtp294505202-8078) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:60309/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
[junit4] 2> 1039143 INFO (qtp1284525073-8017) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={_stateVer_=collection1:9&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 6
[junit4] 2> 1039146 INFO (qtp1284525073-8011) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 1039147 INFO (qtp24934752-8045) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 1039148 INFO (qtp294505202-8080) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 1041162 INFO (qtp1186743350-7974) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1557509815387815936)} 0 2
[junit4] 2> 1041167 INFO (qtp294505202-8081) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1557509815390961664&distrib.from=http://127.0.0.1:60309/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557509815390961664)} 0 1
[junit4] 2> 1041167 INFO (qtp24934752-8046) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1557509815390961664&distrib.from=http://127.0.0.1:60309/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557509815390961664)} 0 1
[junit4] 2> 1041168 INFO (qtp1284525073-8014) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1557509815390961664)} 0 4
[junit4] 2> 1041184 INFO (qtp294505202-8082) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:60309/collection1/&wt=javabin&version=2}{add=[0 (1557509815402496000)]} 0 4
[...truncated too long message...]
re_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Skipping recovery because Solr is shutdown
[junit4] 2> 1228046 INFO (zkCallback-1514-thread-4-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1232218 INFO (recoveryExecutor-1512-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy RecoveryStrategy has been closed
[junit4] 2> 1232218 INFO (recoveryExecutor-1512-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[false]
[junit4] 2> 1232218 INFO (recoveryExecutor-1512-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@3f98865
[junit4] 2> 1232219 WARN (recoveryExecutor-1512-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Stopping recovery for core=[collection1] coreNodeName=[core_node2]
[junit4] 2> 1232219 INFO (zkCallback-1514-thread-3-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1232250 INFO (recoveryExecutor-1512-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
[junit4] 2> 1232250 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.Overseer Overseer (id=97344362028793874-127.0.0.1:60313_-n_0000000004) closing
[junit4] 2> 1232251 INFO (OverseerStateUpdate-97344362028793874-127.0.0.1:60313_-n_0000000004) [n:127.0.0.1:60313_ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:60313_
[junit4] 2> 1232252 WARN (zkCallback-1514-thread-3-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1232253 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 1232254 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@1bbcfd62{HTTP/1.1,[http/1.1]}{127.0.0.1:60313}
[junit4] 2> 1232254 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1f673412{/,null,UNAVAILABLE}
[junit4] 2> 1232255 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 60318
[junit4] 2> 1232257 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60300 60300
[junit4] 2> 1232274 INFO (Thread-1655) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60300 60300
[junit4] 2> 1236047 WARN (Thread-1655) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 5 /solr/security.json
[junit4] 2> 5 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 4 /solr/collections/collection1/state.json
[junit4] 2> 2 /solr/collections/collection1/leader_elect/shard1/election/97344362028793865-core_node1-n_0000000000
[junit4] 2> 2 /solr/overseer_elect/election/97344362028793865-127.0.0.1:60309_-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 207 /solr/overseer/collection-queue-work
[junit4] 2> 48 /solr/overseer/queue
[junit4] 2> 16 /solr/overseer/queue-work
[junit4] 2> 6 /solr/collections
[junit4] 2> 5 /solr/live_nodes
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=PeerSyncReplicationTest -Dtests.method=test -Dtests.seed=A06A4D09107E4409 -Dtests.slow=true -Dtests.locale=ar-DZ -Dtests.timezone=PRC -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] FAILURE 217s J0 | PeerSyncReplicationTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: timeout waiting to see all nodes active
[junit4] > at __randomizedtesting.SeedInfo.seed([A06A4D09107E4409:283E72D3BE8229F1]:0)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 1236113 INFO (SUITE-PeerSyncReplicationTest-seed#[A06A4D09107E4409]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001
[junit4] 2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=7, maxDocsPerChunk=145, blockSize=6), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=7, blockSize=6)), sim=RandomSimilarity(queryNorm=true): {}, locale=ar-DZ, timezone=PRC
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=58012672,total=385875968
[junit4] 2> NOTE: All tests run in this JVM: [TestRuleBasedAuthorizationPlugin, TestStressReorder, TestPointFields, SolrRequestParserTest, PeerSyncTest, TestDocBasedVersionConstraints, TestStressUserVersions, OverseerStatusTest, DistributedQueryComponentOptimizationTest, TestAnalyzeInfixSuggestions, TestPerFieldSimilarity, ReplicationFactorTest, DocValuesTest, SolrCoreTest, DirectUpdateHandlerTest, SparseHLLTest, ClassificationUpdateProcessorFactoryTest, TestLegacyFieldCache, ZkNodePropsTest, TestHdfsUpdateLog, TestJsonFacetRefinement, TestOverriddenPrefixQueryForCustomFieldType, TestDFISimilarityFactory, BlockJoinFacetDistribTest, PreAnalyzedFieldManagedSchemaCloudTest, TestExactSharedStatsCache, TestSQLHandlerNonCloud, SolrCloudExampleTest, SolrGangliaReporterTest, DistributedQueryElevationComponentTest, TestSolrCloudSnapshots, TestSSLRandomization, TestFieldCacheVsDocValues, TestWordDelimiterFilterFactory, TestDynamicLoading, NumericFieldsTest, TestSurroundQueryParser, HdfsCollectionsAPIDistributedZkTest, SecurityConfHandlerTest, CdcrRequestHandlerTest, TestFastLRUCache, TestPushWriter, TestFilteredDocIdSet, TestHashQParserPlugin, DateFieldTest, MultiTermTest, TestDistributedSearch, TestSolrCloudWithSecureImpersonation, RollingRestartTest, HdfsChaosMonkeyNothingIsSafeTest, TestFastOutputStream, HdfsRecoverLeaseTest, JSONWriterTest, ZkCLITest, DOMUtilTest, TestDownShardTolerantSearch, ZkStateReaderTest, DirectSolrConnectionTest, TestAuthorizationFramework, AnalysisAfterCoreReloadTest, SOLR749Test, XmlUpdateRequestHandlerTest, FileUtilsTest, TestSubQueryTransformerCrossCore, TestBinaryResponseWriter, AssignTest, OverseerTest, TestBM25SimilarityFactory, TestDefaultStatsCache, TestMiniSolrCloudClusterSSL, TestTolerantUpdateProcessorRandomCloud, TestFaceting, OverriddenZkACLAndCredentialsProvidersTest, UnloadDistributedZkTest, TestRTGBase, DocExpirationUpdateProcessorFactoryTest, QueryEqualityTest, TestRequestForwarding, StandardRequestHandlerTest, TestSweetSpotSimilarityFactory, PrimUtilsTest, TestHighlightDedupGrouping, PolyFieldTest, TestChildDocTransformer, CopyFieldTest, DistributedFacetPivotWhiteBoxTest, TestReloadDeadlock, TestDefaultSearchFieldResource, TestFieldResource, TestReplicationHandlerBackup, TestCollationField, TestFieldCacheReopen, CdcrUpdateLogTest, ExitableDirectoryReaderTest, TestHashPartitioner, LeaderInitiatedRecoveryOnCommitTest, SearchHandlerTest, RemoteQueryErrorTest, TestFileDictionaryLookup, ShardRoutingTest, CollectionsAPIAsyncDistributedZkTest, TestPhraseSuggestions, TestCSVResponseWriter, SuggesterFSTTest, TestExtendedDismaxParser, TestCSVLoader, CustomCollectionTest, QueryElevationComponentTest, TestPostingsSolrHighlighter, PeerSyncReplicationTest]
[junit4] Completed [238/680 (1!)] on J0 in 216.93s, 1 test, 1 failure <<< FAILURES!
[...truncated 63827 lines...]
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3796
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3796/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC
1 tests failed.
FAILED: org.apache.solr.cloud.PeerSyncReplicationTest.test
Error Message:
timeout waiting to see all nodes active
Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
at __randomizedtesting.SeedInfo.seed([8A52E9F8E7C49C15:206D6224938F1ED]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 11774 lines...]
[junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/init-core-data-001
[junit4] 2> 1792538 INFO (SUITE-PeerSyncReplicationTest-seed#[8A52E9F8E7C49C15]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields
[junit4] 2> 1792543 INFO (SUITE-PeerSyncReplicationTest-seed#[8A52E9F8E7C49C15]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 1792543 INFO (SUITE-PeerSyncReplicationTest-seed#[8A52E9F8E7C49C15]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /l_d/p
[junit4] 2> 1792546 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1792546 INFO (Thread-2528) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1792546 INFO (Thread-2528) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1792647 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.ZkTestServer start zk server on port:55886
[junit4] 2> 1792897 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1792903 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 1792907 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1792911 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1792915 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1792918 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1792921 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1792925 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1792928 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1792931 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1792934 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1793321 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/control-001/cores/collection1
[junit4] 2> 1793324 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1793326 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@a747d51{/l_d/p,null,AVAILABLE}
[junit4] 2> 1793327 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@47d24f1e{HTTP/1.1,[http/1.1]}{127.0.0.1:55889}
[junit4] 2> 1793327 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.Server Started @1799013ms
[junit4] 2> 1793327 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/tempDir-001/control/data, hostContext=/l_d/p, hostPort=55889, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/control-001/cores}
[junit4] 2> 1793328 ERROR (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1793328 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 1793328 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1793329 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1793329 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-25T03:29:12.109Z
[junit4] 2> 1793334 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1793334 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/control-001/solr.xml
[junit4] 2> 1793347 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55886/solr
[junit4] 2> 1793403 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [n:127.0.0.1:55889_l_d%2Fp ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:55889_l_d%2Fp
[junit4] 2> 1793406 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [n:127.0.0.1:55889_l_d%2Fp ] o.a.s.c.Overseer Overseer (id=97341600650297348-127.0.0.1:55889_l_d%2Fp-n_0000000000) starting
[junit4] 2> 1793421 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [n:127.0.0.1:55889_l_d%2Fp ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55889_l_d%2Fp
[junit4] 2> 1793425 INFO (zkCallback-2889-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1793503 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [n:127.0.0.1:55889_l_d%2Fp ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/control-001/cores
[junit4] 2> 1793503 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [n:127.0.0.1:55889_l_d%2Fp ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1793507 INFO (OverseerStateUpdate-97341600650297348-127.0.0.1:55889_l_d%2Fp-n_0000000000) [n:127.0.0.1:55889_l_d%2Fp ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1794524 INFO (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1794540 INFO (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1794646 WARN (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1794649 INFO (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1794677 INFO (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 1794678 INFO (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/control-001/cores/collection1/data/]
[junit4] 2> 1794678 INFO (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@574b7c7c
[junit4] 2> 1794681 INFO (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10103222138828949]
[junit4] 2> 1794697 WARN (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1794715 INFO (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1794715 INFO (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1794716 INFO (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1794716 INFO (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1794717 INFO (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=76.291015625, floorSegmentMB=1.9951171875, forceMergeDeletesPctAllowed=5.914618076587409, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4872145110428895
[junit4] 2> 1794718 INFO (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1422b393[collection1] main]
[junit4] 2> 1794720 INFO (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1794721 INFO (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1794721 INFO (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1794722 INFO (searcherExecutor-5532-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp x:collection1 c:control_collection) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@1422b393[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1794724 INFO (coreLoadExecutor-5531-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557465612685410304
[junit4] 2> 1794736 INFO (coreZkRegister-5524-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp x:collection1 c:control_collection) [n:127.0.0.1:55889_l_d%2Fp c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1794736 INFO (coreZkRegister-5524-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp x:collection1 c:control_collection) [n:127.0.0.1:55889_l_d%2Fp c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1794736 INFO (coreZkRegister-5524-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp x:collection1 c:control_collection) [n:127.0.0.1:55889_l_d%2Fp c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:55889/l_d/p/collection1/
[junit4] 2> 1794736 INFO (coreZkRegister-5524-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp x:collection1 c:control_collection) [n:127.0.0.1:55889_l_d%2Fp c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1794736 INFO (coreZkRegister-5524-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp x:collection1 c:control_collection) [n:127.0.0.1:55889_l_d%2Fp c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:55889/l_d/p/collection1/ has no replicas
[junit4] 2> 1794744 INFO (coreZkRegister-5524-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp x:collection1 c:control_collection) [n:127.0.0.1:55889_l_d%2Fp c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:55889/l_d/p/collection1/ shard1
[junit4] 2> 1794912 INFO (coreZkRegister-5524-thread-1-processing-n:127.0.0.1:55889_l_d%2Fp x:collection1 c:control_collection) [n:127.0.0.1:55889_l_d%2Fp c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1795117 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1795119 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55886/solr ready
[junit4] 2> 1795119 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1795119 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
[junit4] 2> 1795452 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-1-001/cores/collection1
[junit4] 2> 1795453 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-1-001
[junit4] 2> 1795454 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1795456 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@192bb7f9{/l_d/p,null,AVAILABLE}
[junit4] 2> 1795457 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5bf195f2{HTTP/1.1,[http/1.1]}{127.0.0.1:55896}
[junit4] 2> 1795457 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.Server Started @1801143ms
[junit4] 2> 1795457 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/l_d/p, hostPort=55896, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-1-001/cores}
[junit4] 2> 1795458 ERROR (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1795458 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 1795458 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1795458 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1795458 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-25T03:29:14.238Z
[junit4] 2> 1795462 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1795462 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-1-001/solr.xml
[junit4] 2> 1795488 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55886/solr
[junit4] 2> 1795508 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1795519 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55896_l_d%2Fp
[junit4] 2> 1795523 INFO (zkCallback-2893-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1795524 INFO (zkCallback-2899-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1795525 INFO (zkCallback-2889-thread-3-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1795569 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-1-001/cores
[junit4] 2> 1795569 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1795573 INFO (OverseerStateUpdate-97341600650297348-127.0.0.1:55889_l_d%2Fp-n_0000000000) [n:127.0.0.1:55889_l_d%2Fp ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1795684 INFO (zkCallback-2899-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1796588 INFO (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1796601 INFO (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1796704 WARN (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1796706 INFO (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1796731 INFO (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 1796731 INFO (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 1796731 INFO (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@574b7c7c
[junit4] 2> 1796735 INFO (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10103222138828949]
[junit4] 2> 1796760 WARN (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1796780 INFO (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1796780 INFO (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1796781 INFO (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1796781 INFO (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1796782 INFO (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=76.291015625, floorSegmentMB=1.9951171875, forceMergeDeletesPctAllowed=5.914618076587409, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4872145110428895
[junit4] 2> 1796783 INFO (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@41d1cec7[collection1] main]
[junit4] 2> 1796785 INFO (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1796786 INFO (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1796786 INFO (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1796788 INFO (searcherExecutor-5543-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp x:collection1 c:collection1) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@41d1cec7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1796789 INFO (coreLoadExecutor-5542-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557465614850719744
[junit4] 2> 1796802 INFO (coreZkRegister-5537-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp x:collection1 c:collection1) [n:127.0.0.1:55896_l_d%2Fp c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1796802 INFO (coreZkRegister-5537-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp x:collection1 c:collection1) [n:127.0.0.1:55896_l_d%2Fp c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1796802 INFO (coreZkRegister-5537-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp x:collection1 c:collection1) [n:127.0.0.1:55896_l_d%2Fp c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:55896/l_d/p/collection1/
[junit4] 2> 1796802 INFO (coreZkRegister-5537-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp x:collection1 c:collection1) [n:127.0.0.1:55896_l_d%2Fp c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1796802 INFO (coreZkRegister-5537-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp x:collection1 c:collection1) [n:127.0.0.1:55896_l_d%2Fp c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:55896/l_d/p/collection1/ has no replicas
[junit4] 2> 1796810 INFO (coreZkRegister-5537-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp x:collection1 c:collection1) [n:127.0.0.1:55896_l_d%2Fp c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:55896/l_d/p/collection1/ shard1
[junit4] 2> 1796916 INFO (zkCallback-2899-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1796963 INFO (coreZkRegister-5537-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp x:collection1 c:collection1) [n:127.0.0.1:55896_l_d%2Fp c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1797071 INFO (zkCallback-2899-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1797438 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-2-001/cores/collection1
[junit4] 2> 1797439 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-2-001
[junit4] 2> 1797439 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1797441 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6ce43ba7{/l_d/p,null,AVAILABLE}
[junit4] 2> 1797442 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@36e105d5{HTTP/1.1,[http/1.1]}{127.0.0.1:55900}
[junit4] 2> 1797442 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.Server Started @1803128ms
[junit4] 2> 1797442 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/l_d/p, hostPort=55900, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-2-001/cores}
[junit4] 2> 1797442 ERROR (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1797443 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 1797443 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1797443 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1797443 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-25T03:29:16.223Z
[junit4] 2> 1797447 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1797447 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-2-001/solr.xml
[junit4] 2> 1797486 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55886/solr
[junit4] 2> 1797506 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [n:127.0.0.1:55900_l_d%2Fp ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1797516 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [n:127.0.0.1:55900_l_d%2Fp ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55900_l_d%2Fp
[junit4] 2> 1797518 INFO (zkCallback-2893-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1797520 INFO (zkCallback-2899-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1797520 INFO (zkCallback-2905-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1797520 INFO (zkCallback-2889-thread-2-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1797566 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [n:127.0.0.1:55900_l_d%2Fp ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-2-001/cores
[junit4] 2> 1797567 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [n:127.0.0.1:55900_l_d%2Fp ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1797571 INFO (OverseerStateUpdate-97341600650297348-127.0.0.1:55889_l_d%2Fp-n_0000000000) [n:127.0.0.1:55889_l_d%2Fp ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1797740 INFO (zkCallback-2905-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1797740 INFO (zkCallback-2899-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1798590 INFO (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1798603 INFO (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1798692 WARN (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1798694 INFO (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1798717 INFO (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 1798717 INFO (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 1798717 INFO (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@574b7c7c
[junit4] 2> 1798722 INFO (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10103222138828949]
[junit4] 2> 1798735 WARN (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1798754 INFO (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1798754 INFO (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1798756 INFO (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1798757 INFO (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1798757 INFO (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=76.291015625, floorSegmentMB=1.9951171875, forceMergeDeletesPctAllowed=5.914618076587409, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4872145110428895
[junit4] 2> 1798758 INFO (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@72174977[collection1] main]
[junit4] 2> 1798760 INFO (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1798760 INFO (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1798760 INFO (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1798761 INFO (searcherExecutor-5554-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 c:collection1) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@72174977[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1798762 INFO (coreLoadExecutor-5553-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557465616919560192
[junit4] 2> 1798768 INFO (coreZkRegister-5548-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 c:collection1) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 1798769 INFO (updateExecutor-2902-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 1798769 INFO (recoveryExecutor-2903-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 1798769 INFO (recoveryExecutor-2903-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 1798769 INFO (recoveryExecutor-2903-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 1798769 INFO (recoveryExecutor-2903-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 1798769 INFO (recoveryExecutor-2903-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:55896/l_d/p/collection1/] and I am [http://127.0.0.1:55900/l_d/p/collection1/]
[junit4] 2> 1798772 INFO (recoveryExecutor-2903-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:55896/l_d/p]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:55900_l_d%252Fp&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 1798775 INFO (qtp664787721-14841) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 1798776 INFO (qtp664787721-14841) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 1798776 INFO (qtp664787721-14841) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:55900_l_d%2Fp, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:55900/l_d/p","node_name":"127.0.0.1:55900_l_d%2Fp","state":"down"}
[junit4] 2> 1798878 INFO (zkCallback-2899-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1798878 INFO (zkCallback-2905-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1799577 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-3-001/cores/collection1
[junit4] 2> 1799592 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-3-001
[junit4] 2> 1799593 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1799596 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@13f87ea2{/l_d/p,null,AVAILABLE}
[junit4] 2> 1799597 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@605c8b75{HTTP/1.1,[http/1.1]}{127.0.0.1:55916}
[junit4] 2> 1799597 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.Server Started @1805283ms
[junit4] 2> 1799597 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/l_d/p, hostPort=55916, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-3-001/cores}
[junit4] 2> 1799597 ERROR (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1799598 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 1799598 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1799598 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1799598 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-25T03:29:18.378Z
[junit4] 2> 1799602 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1799602 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-3-001/solr.xml
[junit4] 2> 1799616 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55886/solr
[junit4] 2> 1799635 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [n:127.0.0.1:55916_l_d%2Fp ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1799648 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [n:127.0.0.1:55916_l_d%2Fp ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55916_l_d%2Fp
[junit4] 2> 1799651 INFO (zkCallback-2889-thread-2-processing-n:127.0.0.1:55889_l_d%2Fp) [n:127.0.0.1:55889_l_d%2Fp ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1799652 INFO (zkCallback-2912-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1799652 INFO (zkCallback-2905-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1799651 INFO (zkCallback-2893-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1799651 INFO (zkCallback-2899-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1799686 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [n:127.0.0.1:55916_l_d%2Fp ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-3-001/cores
[junit4] 2> 1799686 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [n:127.0.0.1:55916_l_d%2Fp ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1799690 INFO (OverseerStateUpdate-97341600650297348-127.0.0.1:55889_l_d%2Fp-n_0000000000) [n:127.0.0.1:55889_l_d%2Fp ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1799777 INFO (qtp664787721-14841) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:55900_l_d%2Fp, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:55900/l_d/p","node_name":"127.0.0.1:55900_l_d%2Fp","state":"recovering"}
[junit4] 2> 1799777 INFO (qtp664787721-14841) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 1799777 INFO (qtp664787721-14841) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:55900_l_d%252Fp&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1002
[junit4] 2> 1799796 INFO (zkCallback-2905-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1799796 INFO (zkCallback-2899-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1799797 INFO (zkCallback-2912-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1800710 INFO (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1800724 INFO (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1800827 WARN (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1800829 INFO (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1800850 INFO (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 1800850 INFO (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 1800851 INFO (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@574b7c7c
[junit4] 2> 1800857 INFO (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10103222138828949]
[junit4] 2> 1800870 WARN (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1800891 INFO (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1800891 INFO (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1800893 INFO (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1800893 INFO (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1800893 INFO (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=76.291015625, floorSegmentMB=1.9951171875, forceMergeDeletesPctAllowed=5.914618076587409, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4872145110428895
[junit4] 2> 1800894 INFO (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6d1cb0ed[collection1] main]
[junit4] 2> 1800896 INFO (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1800897 INFO (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1800897 INFO (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1800898 INFO (searcherExecutor-5565-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 c:collection1) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6d1cb0ed[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1800899 INFO (coreLoadExecutor-5564-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557465619160367104
[junit4] 2> 1800905 INFO (coreZkRegister-5559-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 c:collection1) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 1800906 INFO (updateExecutor-2909-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 1800906 INFO (recoveryExecutor-2910-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 1800906 INFO (recoveryExecutor-2910-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 1800906 INFO (recoveryExecutor-2910-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 1800906 INFO (recoveryExecutor-2910-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 1800906 INFO (recoveryExecutor-2910-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:55896/l_d/p/collection1/] and I am [http://127.0.0.1:55916/l_d/p/collection1/]
[junit4] 2> 1800909 INFO (recoveryExecutor-2910-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:55896/l_d/p]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:55916_l_d%252Fp&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 1800911 INFO (qtp664787721-14845) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 1800911 INFO (qtp664787721-14845) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 1800911 INFO (qtp664787721-14845) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:55916_l_d%2Fp, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:55916/l_d/p","node_name":"127.0.0.1:55916_l_d%2Fp","state":"down"}
[junit4] 2> 1801014 INFO (zkCallback-2905-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1801014 INFO (zkCallback-2899-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1801014 INFO (zkCallback-2912-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1801290 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 1801290 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30 for each attempt
[junit4] 2> 1801290 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
[junit4] 2> 1801914 INFO (qtp664787721-14845) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:55916_l_d%2Fp, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:55916/l_d/p","node_name":"127.0.0.1:55916_l_d%2Fp","state":"recovering"}
[junit4] 2> 1801914 INFO (qtp664787721-14845) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 1801914 INFO (qtp664787721-14845) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:55916_l_d%252Fp&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1003
[junit4] 2> 1806788 INFO (recoveryExecutor-2903-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:55896/l_d/p/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 1806788 INFO (recoveryExecutor-2903-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:55900/l_d/p START replicas=[http://127.0.0.1:55896/l_d/p/collection1/] nUpdates=1000
[junit4] 2> 1806791 INFO (qtp664787721-14842) [n:127.0.0.1:55896_l_d%2Fp c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1806791 INFO (qtp664787721-14842) [n:127.0.0.1:55896_l_d%2Fp c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp=/l_d/p path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1806792 INFO (recoveryExecutor-2903-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1806792 INFO (recoveryExecutor-2903-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 1806792 INFO (recoveryExecutor-2903-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1806792 INFO (recoveryExecutor-2903-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1806792 INFO (recoveryExecutor-2903-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1806792 INFO (recoveryExecutor-2903-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 1806792 INFO (recoveryExecutor-2903-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 1806792 INFO (recoveryExecutor-2903-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 1806792 INFO (recoveryExecutor-2903-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 1806798 INFO (zkCallback-2905-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1806798 INFO (zkCallback-2899-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1806799 INFO (zkCallback-2912-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1808925 INFO (recoveryExecutor-2910-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:55896/l_d/p/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 1808925 INFO (recoveryExecutor-2910-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:55916/l_d/p START replicas=[http://127.0.0.1:55896/l_d/p/collection1/] nUpdates=1000
[junit4] 2> 1808927 INFO (qtp664787721-14843) [n:127.0.0.1:55896_l_d%2Fp c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1808927 INFO (qtp664787721-14843) [n:127.0.0.1:55896_l_d%2Fp c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp=/l_d/p path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1808927 INFO (recoveryExecutor-2910-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1808928 INFO (recoveryExecutor-2910-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 1808928 INFO (recoveryExecutor-2910-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1808928 INFO (recoveryExecutor-2910-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1808928 INFO (recoveryExecutor-2910-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1808928 INFO (recoveryExecutor-2910-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 1808928 INFO (recoveryExecutor-2910-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 1808928 INFO (recoveryExecutor-2910-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 1808928 INFO (recoveryExecutor-2910-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 1808934 INFO (zkCallback-2905-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1808934 INFO (zkCallback-2899-thread-1-processing-n:127.0.0.1:55896_l_d%2Fp) [n:127.0.0.1:55896_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1808934 INFO (zkCallback-2912-thread-1-processing-n:127.0.0.1:55916_l_d%2Fp) [n:127.0.0.1:55916_l_d%2Fp ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1809331 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
[junit4] 2> 1809334 INFO (qtp830088710-14801) [n:127.0.0.1:55889_l_d%2Fp c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1809334 INFO (qtp830088710-14801) [n:127.0.0.1:55889_l_d%2Fp c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1809335 INFO (qtp830088710-14801) [n:127.0.0.1:55889_l_d%2Fp c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1809335 INFO (qtp830088710-14801) [n:127.0.0.1:55889_l_d%2Fp c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/l_d/p path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 1
[junit4] 2> 1809340 INFO (qtp664787721-14841) [n:127.0.0.1:55896_l_d%2Fp c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1809340 INFO (qtp664787721-14841) [n:127.0.0.1:55896_l_d%2Fp c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1809341 INFO (qtp664787721-14841) [n:127.0.0.1:55896_l_d%2Fp c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1809342 INFO (qtp664787721-14841) [n:127.0.0.1:55896_l_d%2Fp c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/l_d/p path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:55896/l_d/p/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 3
[junit4] 2> 1809343 INFO (qtp1737039614-14904) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1809343 INFO (qtp1954238832-14869) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1809343 INFO (qtp1954238832-14869) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1809343 INFO (qtp1737039614-14904) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1809345 INFO (qtp1954238832-14869) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1809345 INFO (qtp1737039614-14904) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1809345 INFO (qtp1954238832-14869) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/l_d/p path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:55896/l_d/p/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 2
[junit4] 2> 1809345 INFO (qtp1737039614-14904) [n:127.0.0.1:55916_l_d%2Fp c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/l_d/p path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:55896/l_d/p/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 2
[junit4] 2> 1809346 INFO (qtp664787721-14846) [n:127.0.0.1:55896_l_d%2Fp c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/l_d/p path=/update params={_stateVer_=collection1:9&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 8
[junit4] 2> 1809349 INFO (qtp664787721-14842) [n:127.0.0.1:55896_l_d%2Fp c:collection1 s:shard1 r:c
[...truncated too long message...]
ccurred - updating... (live nodes size: [1])
[junit4] 2> 2000980 INFO (recoveryExecutor-2917-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy RecoveryStrategy has been closed
[junit4] 2> 2000980 INFO (recoveryExecutor-2917-thread-1-processing-n:127.0.0.1:55900_l_d%2Fp x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[false]
[junit4] 2> 2000980 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.Overseer Overseer (id=97341600650297362-127.0.0.1:55900_l_d%2Fp-n_0000000004) closing
[junit4] 2> 2000980 INFO (zkCallback-2919-thread-3-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@40fe5aa5
[junit4] 2> 2000981 INFO (OverseerStateUpdate-97341600650297362-127.0.0.1:55900_l_d%2Fp-n_0000000004) [n:127.0.0.1:55900_l_d%2Fp ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:55900_l_d%2Fp
[junit4] 2> 2000981 WARN (zkCallback-2919-thread-3-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Stopping recovery for core=[collection1] coreNodeName=[core_node2]
[junit4] 2> 2001116 INFO (zkCallback-2919-thread-3-processing-n:127.0.0.1:55900_l_d%2Fp) [n:127.0.0.1:55900_l_d%2Fp c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
[junit4] 2> 2001117 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 2001118 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@46081837{HTTP/1.1,[http/1.1]}{127.0.0.1:55900}
[junit4] 2> 2001119 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3838d3e7{/l_d/p,null,UNAVAILABLE}
[junit4] 2> 2001120 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 55916
[junit4] 2> 2001122 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A52E9F8E7C49C15]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:55886 55886
[junit4] 2> 2001139 INFO (Thread-2528) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:55886 55886
[junit4] 2> 2003023 WARN (Thread-2528) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 5 /solr/security.json
[junit4] 2> 5 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 4 /solr/collections/collection1/state.json
[junit4] 2> 2 /solr/overseer_elect/election/97341600650297353-127.0.0.1:55896_l_d%2Fp-n_0000000001
[junit4] 2> 2 /solr/collections/collection1/leader_elect/shard1/election/97341600650297353-core_node1-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 207 /solr/overseer/collection-queue-work
[junit4] 2> 43 /solr/overseer/queue
[junit4] 2> 15 /solr/overseer/queue-work
[junit4] 2> 6 /solr/collections
[junit4] 2> 5 /solr/live_nodes
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=PeerSyncReplicationTest -Dtests.method=test -Dtests.seed=8A52E9F8E7C49C15 -Dtests.slow=true -Dtests.locale=sr-Latn-BA -Dtests.timezone=Asia/Barnaul -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 211s J0 | PeerSyncReplicationTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: timeout waiting to see all nodes active
[junit4] > at __randomizedtesting.SeedInfo.seed([8A52E9F8E7C49C15:206D6224938F1ED]:0)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2003096 INFO (SUITE-PeerSyncReplicationTest-seed#[8A52E9F8E7C49C15]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_8A52E9F8E7C49C15-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=true): {}, locale=sr-Latn-BA, timezone=Asia/Barnaul
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=221464480,total=381157376
[junit4] 2> NOTE: All tests run in this JVM: [TestDynamicLoading, TestBinaryResponseWriter, TestRandomFlRTGCloud, SolrGangliaReporterTest, DirectSolrConnectionTest, JavabinLoaderTest, TestManagedResource, QueryResultKeyTest, TestUpdate, StressHdfsTest, TestStressVersions, ExplicitHLLTest, SharedFSAutoReplicaFailoverTest, DistributedQueueTest, TestFaceting, QueryElevationComponentTest, HdfsCollectionsAPIDistributedZkTest, PrimUtilsTest, DistributedFacetPivotSmallAdvancedTest, SolrCloudExampleTest, TestLegacyNumericRangeQueryBuilder, TestSolrCLIRunExample, AnalysisAfterCoreReloadTest, TestClusterProperties, TestRandomCollapseQParserPlugin, TestSweetSpotSimilarityFactory, ClassificationUpdateProcessorTest, TestSchemaVersionResource, BigEndianAscendingWordSerializerTest, SecurityConfHandlerTest, TestJsonRequest, TestMiniSolrCloudClusterSSL, PluginInfoTest, TestSolrCloudWithSecureImpersonation, TestConfigOverlay, TestFieldCacheSort, AddSchemaFieldsUpdateProcessorFactoryTest, HdfsUnloadDistributedZkTest, SolrCoreTest, ClusterStateUpdateTest, TestDeleteCollectionOnDownNodes, PingRequestHandlerTest, TestConfigReload, TestHdfsCloudBackupRestore, CopyFieldTest, TestManagedSchemaAPI, SolrPluginUtilsTest, AlternateDirectoryTest, SoftAutoCommitTest, TriLevelCompositeIdRoutingTest, HighlighterTest, SortSpecParsingTest, TestPhraseSuggestions, ParsingFieldUpdateProcessorsTest, TestDocumentBuilder, SynonymTokenizerTest, TestDefaultStatsCache, TestMiniSolrCloudCluster, MigrateRouteKeyTest, BasicAuthStandaloneTest, DateFieldTest, TestReplicationHandlerBackup, ZkCLITest, JSONWriterTest, UnloadDistributedZkTest, LeaderInitiatedRecoveryOnCommitTest, TestFieldTypeResource, XsltUpdateRequestHandlerTest, HttpPartitionTest, TestStressLiveNodes, TestStressLucene, TestRuleBasedAuthorizationPlugin, ResponseHeaderTest, SimpleMLTQParserTest, TestPostingsSolrHighlighter, TestUnifiedSolrHighlighter, ShufflingReplicaListTransformerTest, FacetPivotSmallTest, SyncSliceTest, TestGraphMLResponseWriter, SolrJmxReporterTest, PeerSyncTest, TestFieldResource, DocValuesTest, TestJsonFacetRefinement, DirectUpdateHandlerTest, SolrTestCaseJ4Test, DisMaxRequestHandlerTest, SuggesterFSTTest, TestSolrJ, ZkStateReaderTest, TermsComponentTest, DocumentAnalysisRequestHandlerTest, TestMergePolicyConfig, CustomCollectionTest, TestConfigSetsAPI, TestSurroundQueryParser, TestCSVResponseWriter, TestCloudInspectUtil, TestInfoStreamLogging, ShardRoutingTest, SaslZkACLProviderTest, TestTolerantSearch, LeaderElectionTest, TestSolrConfigHandlerConcurrent, TestSolrQueryParserDefaultOperatorResource, InfoHandlerTest, FileUtilsTest, RulesTest, LeaderFailoverAfterPartitionTest, DistributedFacetPivotWhiteBoxTest, ReplicationFactorTest, TestHighlightDedupGrouping, TestSortByMinMaxFunction, TestFastLRUCache, TestRebalanceLeaders, PreAnalyzedFieldManagedSchemaCloudTest, SchemaVersionSpecificBehaviorTest, DeleteStatusTest, MultiThreadedOCPTest, MoreLikeThisHandlerTest, TestSimpleQParserPlugin, TestRTGBase, CdcrReplicationDistributedZkTest, TestShortCircuitedRequests, FastVectorHighlighterTest, TestLFUCache, TestLRUCache, TestFilteredDocIdSet, TestShardHandlerFactory, TestFieldCacheVsDocValues, TestManagedSynonymFilterFactory, BlockJoinFacetDistribTest, ExitableDirectoryReaderTest, TestXmlQParser, TestHighFrequencyDictionaryFactory, TestRequestForwarding, CollectionsAPIAsyncDistributedZkTest, TestSolr4Spatial2, BooleanFieldTest, OverseerTest, TestMinMaxOnMultiValuedField, WrapperMergePolicyFactoryTest, StandardRequestHandlerTest, RequestLoggingTest, TestComplexPhraseLeadingWildcard, ConcurrentDeleteAndCreateCollectionTest, DocValuesMultiTest, TestHdfsUpdateLog, HdfsRecoverLeaseTest, TestSSLRandomization, DeleteLastCustomShardedReplicaTest, TestTolerantUpdateProcessorRandomCloud, TestChildDocTransformer, IndexSchemaRuntimeFieldTest, SolrCoreMetricManagerTest, BaseCdcrDistributedZkTest, SOLR749Test, TestAuthorizationFramework, PeerSyncWithIndexFingerprintCachingTest, PeerSyncReplicationTest]
[junit4] Completed [336/680 (1!)] on J0 in 210.59s, 1 test, 1 failure <<< FAILURES!
[...truncated 63513 lines...]