You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/12/09 18:54:16 UTC
[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 333 -
Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/333/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC
4 tests failed.
FAILED: org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForHashRouter
Error Message:
Collection not found: routeFieldColl
Stack Trace:
org.apache.solr.common.SolrException: Collection not found: routeFieldColl
at __randomizedtesting.SeedInfo.seed([C7823DDAC55E5F3:A44EBD0033340EA9]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:850)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
at org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForHashRouter(CustomCollectionTest.java:166)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForImplicitRouter
Error Message:
Collection not found: withShardField
Stack Trace:
org.apache.solr.common.SolrException: Collection not found: withShardField
at __randomizedtesting.SeedInfo.seed([C7823DDAC55E5F3:5928CB4F00AC2A03]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:850)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
at org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForImplicitRouter(CustomCollectionTest.java:141)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.CustomCollectionTest.testCreateShardRepFactor
Error Message:
Error from server at http://127.0.0.1:51538/solr: Could not find collection : testCreateShardRepFactor
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:51538/solr: Could not find collection : testCreateShardRepFactor
at __randomizedtesting.SeedInfo.seed([C7823DDAC55E5F3:F896A959B401F033]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.cloud.CustomCollectionTest.testCreateShardRepFactor(CustomCollectionTest.java:190)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test
Error Message:
Could not find collection:collection2
Stack Trace:
java.lang.AssertionError: Could not find collection:collection2
at __randomizedtesting.SeedInfo.seed([C7823DDAC55E5F3:842C1C0702A9880B]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNotNull(Assert.java:526)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:140)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:135)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:907)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:612)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12602 lines...]
[junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
[junit4] 2> 1426988 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[C7823DDAC55E5F3]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/init-core-data-001
[junit4] 2> 1426989 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[C7823DDAC55E5F3]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 1426992 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[C7823DDAC55E5F3]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 1426993 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[C7823DDAC55E5F3]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 1426996 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1426996 INFO (Thread-2657) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1426996 INFO (Thread-2657) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1427003 ERROR (Thread-2657) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1427100 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.ZkTestServer start zk server on port:63721
[junit4] 2> 1427119 INFO (zkConnectionManagerCallback-3145-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1427130 INFO (zkConnectionManagerCallback-3147-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1427140 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1427143 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 1427145 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1427147 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1427149 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1427151 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1427152 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1427154 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1427156 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1427157 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1427159 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1427162 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 1427512 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1427513 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3278f1ef{/,null,AVAILABLE}
[junit4] 2> 1427514 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@44b9282c{HTTP/1.1,[http/1.1]}{127.0.0.1:63725}
[junit4] 2> 1427514 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.e.j.s.Server Started @1430894ms
[junit4] 2> 1427514 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/tempDir-001/control/data, replicaType=NRT, hostContext=/, hostPort=63725, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/control-001/cores}
[junit4] 2> 1427516 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1427516 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 1427516 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1427516 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1427516 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-09T17:52:26.331Z
[junit4] 2> 1427518 INFO (zkConnectionManagerCallback-3149-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1427519 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1427519 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/control-001/solr.xml
[junit4] 2> 1427528 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1427528 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1427529 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1427532 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:63721/solr
[junit4] 2> 1427533 INFO (zkConnectionManagerCallback-3153-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1427536 INFO (zkConnectionManagerCallback-3155-thread-1-processing-n:127.0.0.1:63725_) [n:127.0.0.1:63725_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1427667 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63725_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1427668 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63725_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:63725_
[junit4] 2> 1427669 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63725_ ] o.a.s.c.Overseer Overseer (id=99145609760931844-127.0.0.1:63725_-n_0000000000) starting
[junit4] 2> 1427691 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63725_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63725_
[junit4] 2> 1427695 INFO (OverseerStateUpdate-99145609760931844-127.0.0.1:63725_-n_0000000000) [n:127.0.0.1:63725_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1427760 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63725_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506
[junit4] 2> 1427767 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63725_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506
[junit4] 2> 1427767 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63725_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506
[junit4] 2> 1427768 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63725_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/control-001/cores
[junit4] 2> 1427789 INFO (zkConnectionManagerCallback-3161-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1427791 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1427793 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:63721/solr ready
[junit4] 2> 1427800 INFO (qtp807345527-14951) [n:127.0.0.1:63725_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:63725_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1427803 INFO (OverseerThreadFactory-6006-thread-1-processing-n:127.0.0.1:63725_) [n:127.0.0.1:63725_ ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 1427911 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1427912 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1428026 INFO (zkCallback-3154-thread-1-processing-n:127.0.0.1:63725_) [n:127.0.0.1:63725_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1428930 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 1428938 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 1428996 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1429015 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 1429015 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506
[junit4] 2> 1429015 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1429015 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 1429017 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 1429019 WARN (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1429061 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1429061 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1429062 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1429062 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1429063 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=11, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=4.4716796875, floorSegmentMB=1.1171875, forceMergeDeletesPctAllowed=3.4447760693924936, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1429063 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@478516e9[control_collection_shard1_replica_n1] main]
[junit4] 2> 1429065 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1429065 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1429065 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1429066 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1586329758341267456
[junit4] 2> 1429066 INFO (searcherExecutor-6009-thread-1-processing-n:127.0.0.1:63725_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@478516e9[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1429071 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1429072 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1429072 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:63725/control_collection_shard1_replica_n1/
[junit4] 2> 1429072 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1429072 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:63725/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 1429072 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1429077 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:63725/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 1429186 INFO (zkCallback-3154-thread-2-processing-n:127.0.0.1:63725_) [n:127.0.0.1:63725_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1429236 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1429238 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1327
[junit4] 2> 1429245 INFO (qtp807345527-14951) [n:127.0.0.1:63725_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1429340 INFO (zkCallback-3154-thread-2-processing-n:127.0.0.1:63725_) [n:127.0.0.1:63725_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1429807 INFO (OverseerCollectionConfigSetProcessor-99145609760931844-127.0.0.1:63725_-n_0000000000) [n:127.0.0.1:63725_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1430245 INFO (qtp807345527-14951) [n:127.0.0.1:63725_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:63725_&wt=javabin&version=2} status=0 QTime=2445
[junit4] 2> 1430251 INFO (zkConnectionManagerCallback-3165-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1430252 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1430253 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:63721/solr ready
[junit4] 2> 1430253 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1430255 INFO (qtp807345527-14956) [n:127.0.0.1:63725_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1430257 INFO (OverseerThreadFactory-6006-thread-2-processing-n:127.0.0.1:63725_) [n:127.0.0.1:63725_ ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 1430258 WARN (OverseerThreadFactory-6006-thread-2-processing-n:127.0.0.1:63725_) [n:127.0.0.1:63725_ ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 1430475 INFO (qtp807345527-14956) [n:127.0.0.1:63725_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1430476 INFO (qtp807345527-14956) [n:127.0.0.1:63725_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=221
[junit4] 2> 1430737 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/shard-1-001 of type NRT
[junit4] 2> 1430738 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1430739 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@20e455b8{/,null,AVAILABLE}
[junit4] 2> 1430739 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6ba8c86f{HTTP/1.1,[http/1.1]}{127.0.0.1:63741}
[junit4] 2> 1430739 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.e.j.s.Server Started @1434119ms
[junit4] 2> 1430739 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=63741, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/shard-1-001/cores}
[junit4] 2> 1430741 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1430742 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 1430742 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1430742 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1430742 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-09T17:52:29.557Z
[junit4] 2> 1430743 INFO (zkConnectionManagerCallback-3167-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1430744 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1430745 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/shard-1-001/solr.xml
[junit4] 2> 1430749 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1430749 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1430750 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1430753 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:63721/solr
[junit4] 2> 1430754 INFO (zkConnectionManagerCallback-3171-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1430757 INFO (zkConnectionManagerCallback-3173-thread-1-processing-n:127.0.0.1:63741_) [n:127.0.0.1:63741_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1430765 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63741_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1430769 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63741_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1430771 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63741_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63741_
[junit4] 2> 1430773 INFO (zkCallback-3154-thread-2-processing-n:127.0.0.1:63725_) [n:127.0.0.1:63725_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1430775 INFO (zkCallback-3164-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1430778 INFO (zkCallback-3172-thread-1-processing-n:127.0.0.1:63741_) [n:127.0.0.1:63741_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1430848 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63741_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506
[junit4] 2> 1430855 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63741_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506
[junit4] 2> 1430855 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63741_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506
[junit4] 2> 1430855 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63741_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/shard-1-001/cores
[junit4] 2> 1430882 INFO (qtp807345527-14949) [n:127.0.0.1:63725_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:63741_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1430885 INFO (OverseerCollectionConfigSetProcessor-99145609760931844-127.0.0.1:63725_-n_0000000000) [n:127.0.0.1:63725_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1430886 INFO (OverseerThreadFactory-6006-thread-3-processing-n:127.0.0.1:63725_) [n:127.0.0.1:63725_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:63741_ for creating new replica
[junit4] 2> 1430896 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n61&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1430896 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1431004 INFO (zkCallback-3172-thread-1-processing-n:127.0.0.1:63741_) [n:127.0.0.1:63741_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1432507 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 1432521 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.s.IndexSchema [collection1_shard2_replica_n61] Schema name=test
[junit4] 2> 1432580 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1432600 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n61' using configuration from collection collection1, trusted=true
[junit4] 2> 1432600 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n61' (registry 'solr.core.collection1.shard2.replica_n61') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506
[junit4] 2> 1432600 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1432601 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SolrCore [[collection1_shard2_replica_n61] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/shard-1-001/cores/collection1_shard2_replica_n61], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/shard-1-001/cores/collection1_shard2_replica_n61/data/]
[junit4] 2> 1432602 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 1432604 WARN (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1432659 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1432659 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1432662 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1432662 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1432663 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=11, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=4.4716796875, floorSegmentMB=1.1171875, forceMergeDeletesPctAllowed=3.4447760693924936, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1432668 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.s.SolrIndexSearcher Opening [Searcher@5df510b0[collection1_shard2_replica_n61] main]
[junit4] 2> 1432670 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1432671 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1432672 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1432672 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1586329762122432512
[junit4] 2> 1432672 INFO (searcherExecutor-6020-thread-1-processing-n:127.0.0.1:63741_ x:collection1_shard2_replica_n61 s:shard2 c:collection1) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SolrCore [collection1_shard2_replica_n61] Registered new searcher Searcher@5df510b0[collection1_shard2_replica_n61] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1432677 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1432677 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1432677 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:63741/collection1_shard2_replica_n61/
[junit4] 2> 1432677 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1432677 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy http://127.0.0.1:63741/collection1_shard2_replica_n61/ has no replicas
[junit4] 2> 1432677 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1432682 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:63741/collection1_shard2_replica_n61/ shard2
[junit4] 2> 1432784 INFO (zkCallback-3172-thread-1-processing-n:127.0.0.1:63741_) [n:127.0.0.1:63741_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1432834 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1432837 INFO (qtp682045734-14999) [n:127.0.0.1:63741_ c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n61&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1941
[junit4] 2> 1432840 INFO (qtp807345527-14949) [n:127.0.0.1:63725_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:63741_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1957
[junit4] 2> 1432890 INFO (OverseerCollectionConfigSetProcessor-99145609760931844-127.0.0.1:63725_-n_0000000000) [n:127.0.0.1:63725_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1432940 INFO (zkCallback-3172-thread-1-processing-n:127.0.0.1:63741_) [n:127.0.0.1:63741_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1433270 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/shard-2-001 of type NRT
[junit4] 2> 1433271 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1433272 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4e2991fc{/,null,AVAILABLE}
[junit4] 2> 1433272 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@55795e01{HTTP/1.1,[http/1.1]}{127.0.0.1:63758}
[junit4] 2> 1433272 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.e.j.s.Server Started @1436653ms
[junit4] 2> 1433274 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/tempDir-001/jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=63758, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/shard-2-001/cores}
[junit4] 2> 1433274 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1433274 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 1433275 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1433275 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1433275 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-09T17:52:32.090Z
[junit4] 2> 1433276 INFO (zkConnectionManagerCallback-3176-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1433277 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1433277 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/shard-2-001/solr.xml
[junit4] 2> 1433412 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1433412 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1433413 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1433416 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:63721/solr
[junit4] 2> 1433417 INFO (zkConnectionManagerCallback-3180-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1433419 INFO (zkConnectionManagerCallback-3182-thread-1-processing-n:127.0.0.1:63758_) [n:127.0.0.1:63758_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1433428 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63758_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1433432 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63758_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1433434 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63758_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63758_
[junit4] 2> 1433436 INFO (zkCallback-3154-thread-2-processing-n:127.0.0.1:63725_) [n:127.0.0.1:63725_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1433437 INFO (zkCallback-3172-thread-1-processing-n:127.0.0.1:63741_) [n:127.0.0.1:63741_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1433437 INFO (zkCallback-3164-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1433439 INFO (zkCallback-3181-thread-1-processing-n:127.0.0.1:63758_) [n:127.0.0.1:63758_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1433547 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63758_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506
[junit4] 2> 1433554 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63758_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506
[junit4] 2> 1433554 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63758_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506
[junit4] 2> 1433556 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63758_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/shard-2-001/cores
[junit4] 2> 1433586 INFO (qtp807345527-14952) [n:127.0.0.1:63725_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:63758_&action=ADDREPLICA&collection=collection1&shard=shard3&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1433589 INFO (OverseerThreadFactory-6006-thread-4-processing-n:127.0.0.1:63725_) [n:127.0.0.1:63725_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:63758_ for creating new replica
[junit4] 2> 1433598 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard3_replica_n63&action=CREATE&collection=collection1&shard=shard3&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1433599 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1433704 INFO (zkCallback-3172-thread-1-processing-n:127.0.0.1:63741_) [n:127.0.0.1:63741_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1433704 INFO (zkCallback-3181-thread-1-processing-n:127.0.0.1:63758_) [n:127.0.0.1:63758_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1434626 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 1434634 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.s.IndexSchema [collection1_shard3_replica_n63] Schema name=test
[junit4] 2> 1434696 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1434715 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard3_replica_n63' using configuration from collection collection1, trusted=true
[junit4] 2> 1434715 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard3.replica_n63' (registry 'solr.core.collection1.shard3.replica_n63') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506
[junit4] 2> 1434715 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1434715 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SolrCore [[collection1_shard3_replica_n63] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/shard-2-001/cores/collection1_shard3_replica_n63], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/shard-2-001/cores/collection1_shard3_replica_n63/data/]
[junit4] 2> 1434717 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 1434719 WARN (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1434761 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1434761 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1434762 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1434762 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1434763 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=11, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=4.4716796875, floorSegmentMB=1.1171875, forceMergeDeletesPctAllowed=3.4447760693924936, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1434763 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.s.SolrIndexSearcher Opening [Searcher@7c709283[collection1_shard3_replica_n63] main]
[junit4] 2> 1434764 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1434765 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1434765 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1434765 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1586329764317102080
[junit4] 2> 1434766 INFO (searcherExecutor-6031-thread-1-processing-n:127.0.0.1:63758_ x:collection1_shard3_replica_n63 s:shard3 c:collection1) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SolrCore [collection1_shard3_replica_n63] Registered new searcher Searcher@7c709283[collection1_shard3_replica_n63] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1434773 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1434773 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1434773 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:63758/collection1_shard3_replica_n63/
[junit4] 2> 1434773 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1434773 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SyncStrategy http://127.0.0.1:63758/collection1_shard3_replica_n63/ has no replicas
[junit4] 2> 1434773 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1434777 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:63758/collection1_shard3_replica_n63/ shard3
[junit4] 2> 1434882 INFO (zkCallback-3181-thread-1-processing-n:127.0.0.1:63758_) [n:127.0.0.1:63758_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1434882 INFO (zkCallback-3172-thread-1-processing-n:127.0.0.1:63741_) [n:127.0.0.1:63741_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1434937 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1434940 INFO (qtp853039595-15030) [n:127.0.0.1:63758_ c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard3_replica_n63&action=CREATE&collection=collection1&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1341
[junit4] 2> 1434942 INFO (qtp807345527-14952) [n:127.0.0.1:63725_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:63758_&action=ADDREPLICA&collection=collection1&shard=shard3&type=NRT&wt=javabin&version=2} status=0 QTime=1356
[junit4] 2> 1435043 INFO (zkCallback-3181-thread-1-processing-n:127.0.0.1:63758_) [n:127.0.0.1:63758_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1435043 INFO (zkCallback-3172-thread-1-processing-n:127.0.0.1:63741_) [n:127.0.0.1:63741_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1435202 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/shard-3-001 of type NRT
[junit4] 2> 1435203 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1435204 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5e6572a4{/,null,AVAILABLE}
[junit4] 2> 1435204 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@566061c8{HTTP/1.1,[http/1.1]}{127.0.0.1:63766}
[junit4] 2> 1435204 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.e.j.s.Server Started @1438585ms
[junit4] 2> 1435206 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/tempDir-001/jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=63766, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/shard-3-001/cores}
[junit4] 2> 1435207 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1435207 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 1435207 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1435207 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1435207 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-09T17:52:34.022Z
[junit4] 2> 1435208 INFO (zkConnectionManagerCallback-3185-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1435210 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1435210 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/shard-3-001/solr.xml
[junit4] 2> 1435215 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1435215 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1435215 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1435218 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:63721/solr
[junit4] 2> 1435220 INFO (zkConnectionManagerCallback-3189-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1435222 INFO (zkConnectionManagerCallback-3191-thread-1-processing-n:127.0.0.1:63766_) [n:127.0.0.1:63766_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1435231 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63766_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1435234 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63766_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1435237 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63766_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63766_
[junit4] 2> 1435238 INFO (zkCallback-3172-thread-1-processing-n:127.0.0.1:63741_) [n:127.0.0.1:63741_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1435238 INFO (zkCallback-3154-thread-2-processing-n:127.0.0.1:63725_) [n:127.0.0.1:63725_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1435239 INFO (zkCallback-3164-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1435239 INFO (zkCallback-3181-thread-1-processing-n:127.0.0.1:63758_) [n:127.0.0.1:63758_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1435241 INFO (zkCallback-3190-thread-1-processing-n:127.0.0.1:63766_) [n:127.0.0.1:63766_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1435298 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63766_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506
[junit4] 2> 1435305 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63766_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506
[junit4] 2> 1435305 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63766_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3dab2506
[junit4] 2> 1435306 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[C7823DDAC55E5F3]) [n:127.0.0.1:63766_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_C7823DDAC55E5F3-001/shard-3-001/cores
[junit4] 2> 1435340 INFO (qtp807345527-14953) [n:127.0.0.1:63725_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:63766_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1435343 INFO (OverseerCollectionConfigSetProcessor-99145609760931844-127.0.0.1:63725_-n_0000000000) [n:127.0.0.1:63725_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1435344 INFO (OverseerThreadFactory-6006-thread-5-processing-n:127.0.0.1:63725_) [n:127.0.0.1:63725_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:63766_ for creating new replica
[junit4] 2> 1435353 INFO (qtp801311282-15060) [n:127.0.0.1:63766_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n65&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1435353 INFO (qtp801311282-15060) [n:127.0.0.1:63766_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1435461 INFO (zkCallback-3172-thread-1-processing-n:127.0.0.1:63741_) [n:127.0.0.1:63741_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1435461 INFO (zkCallback-3190-thread-1-processing-n:127.0.0.1:63766_) [n:127.0.0.1:63766_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1435461 INFO (zkCallback-3181-thread-1-processing-n:127.0.0.1:63758_) [n:127.0.0.1:63758_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1436377 INFO (qtp801311282-15060) [n:127.0.0.1:63766_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 1436385 INFO (qtp801311282-15060) [n:127.0.0.1:63766_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.s.IndexSchema [collection1_shard1_replica_n65] Schema name=test
[junit4] 2> 1436440 INFO (qtp801311282-15060) [n:127.0.0.1:63766_ c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.s.IndexSchema Loaded schema test/1.6 with
[...truncated too long message...]
t org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2492255 INFO (jetty-closer-4806-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7f0ba9e1{/solr,null,UNAVAILABLE}
[junit4] 2> 2492255 INFO (jetty-closer-4806-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@60049125{/solr,null,UNAVAILABLE}
[junit4] 2> 2492257 ERROR (SUITE-CustomCollectionTest-seed#[C7823DDAC55E5F3]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2492260 INFO (SUITE-CustomCollectionTest-seed#[C7823DDAC55E5F3]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:51535 51535
[junit4] 2> 2492310 INFO (Thread-4280) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:51535 51535
[junit4] 2> 2492435 WARN (Thread-4280) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 9 /solr/aliases.json
[junit4] 2> 9 /solr/clusterprops.json
[junit4] 2> 4 /solr/collections/routeFieldColl/state.json
[junit4] 2> 4 /solr/security.json
[junit4] 2> 4 /solr/configs/conf
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 85 /solr/collections/implicitcoll/state.json
[junit4] 2> 12 /solr/collections/routeFieldColl/state.json
[junit4] 2> 9 /solr/clusterstate.json
[junit4] 2> 2 /solr/overseer_elect/election/99145672438448137-127.0.0.1:51538_solr-n_0000000000
[junit4] 2> 2 /solr/collections/implicitcoll/leader_elect/a/election/99145672438448138-core_node7-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 30 /solr/collections
[junit4] 2> 9 /solr/live_nodes
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_C7823DDAC55E5F3-001
[junit4] 2> Dec 09, 2017 6:10:11 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true): {}, locale=pt, timezone=America/Indianapolis
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=61304160,total=521142272
[junit4] 2> NOTE: All tests run in this JVM: [TestSolrQueryParser, TestCollapseQParserPlugin, BitVectorTest, SystemInfoHandlerTest, OverseerCollectionConfigSetProcessorTest, SimpleMLTQParserTest, TestCryptoKeys, TestSchemaVersionResource, TestUninvertingReader, SolrIndexSplitterTest, TestRestManager, TestManagedSchema, TestExactSharedStatsCache, TestObjectReleaseTracker, SortSpecParsingTest, HttpSolrCallGetCoreTest, AnalyticsMergeStrategyTest, TestBlobHandler, TestAuthorizationFramework, TestFieldCacheSortRandom, TestSSLRandomization, TermVectorComponentDistributedTest, TestSchemaResource, SolrIndexConfigTest, TestFieldSortValues, PolyFieldTest, TestSystemIdResolver, ReplaceNodeTest, DistributedQueryComponentCustomSortTest, TestCSVLoader, TestEmbeddedSolrServerConstructors, TestInPlaceUpdatesDistrib, TestStressVersions, FileUtilsTest, SolrCoreCheckLockOnStartupTest, TestXmlQParser, TestDocTermOrds, HdfsTlogReplayBufferedWhileIndexingTest, TestDFISimilarityFactory, SSLMigrationTest, DistributedIntervalFacetingTest, BasicFunctionalityTest, SimpleFacetsTest, SmileWriterTest, AutoAddReplicasPlanActionTest, ZkSolrClientTest, CoreAdminHandlerTest, TestCrossCoreJoin, SimpleCollectionCreateDeleteTest, DocValuesMultiTest, DocumentAnalysisRequestHandlerTest, TestDelegationWithHadoopAuth, BigEndianAscendingWordDeserializerTest, OverseerTaskQueueTest, HdfsBasicDistributedZkTest, InfoHandlerTest, TestUtilizeNode, TestFunctionQuery, TestStressCloudBlindAtomicUpdates, RegexBytesRefFilterTest, TestFieldCacheWithThreads, SolrCloudExampleTest, BlockJoinFacetSimpleTest, TestSolr4Spatial2, MetricUtilsTest, HighlighterMaxOffsetTest, TestPartialUpdateDeduplication, OverseerRolesTest, LukeRequestHandlerTest, TestDistribDocBasedVersion, PluginInfoTest, UpdateLogTest, TestDefaultStatsCache, CollectionsAPIAsyncDistributedZkTest, PreAnalyzedFieldTest, TestCharFilters, MoreLikeThisHandlerTest, TestFieldCacheReopen, TestShortCircuitedRequests, TestRandomFlRTGCloud, CdcrVersionReplicationTest, CollectionStateFormat2Test, TestJsonFacetRefinement, TimeRoutedAliasUpdateProcessorTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestExceedMaxTermLength, NodeAddedTriggerTest, TestFaceting, TestCoreContainer, MoveReplicaHDFSTest, SpellingQueryConverterTest, TestSubQueryTransformerCrossCore, TestSolrCLIRunExample, PreAnalyzedFieldManagedSchemaCloudTest, TestManagedSchemaThreadSafety, CoreAdminRequestStatusTest, TestReload, TestRuleBasedAuthorizationPlugin, TermsComponentTest, TestHashPartitioner, TestDistributedMissingSort, TestAnalyzeInfixSuggestions, CloudExitableDirectoryReaderTest, HdfsNNFailoverTest, TestFieldResource, SOLR749Test, BaseCdcrDistributedZkTest, BlockDirectoryTest, TestDistributedStatsComponentCardinality, StatelessScriptUpdateProcessorFactoryTest, TestUniqueKeyFieldResource, TestZkChroot, SuggestComponentContextFilterQueryTest, AssignTest, HdfsSyncSliceTest, TestCloudNestedDocsSort, TestLMJelinekMercerSimilarityFactory, TestLegacyNumericUtils, CdcrReplicationHandlerTest, SpatialRPTFieldTypeTest, DataDrivenBlockJoinTest, TestBadConfig, TestLeaderElectionWithEmptyReplica, UUIDUpdateProcessorFallbackTest, DebugComponentTest, RecoveryAfterSoftCommitTest, QueryResultKeyTest, TestTrie, TestDocumentBuilder, ZkFailoverTest, TestInfoStreamLogging, SolrGraphiteReporterTest, InfixSuggestersTest, BadComponentTest, DocumentBuilderTest, UpdateRequestProcessorFactoryTest, TestGeoJSONResponseWriter, AutoscalingHistoryHandlerTest, ComputePlanActionTest, UpdateParamsTest, ExternalFileFieldSortTest, TestSolrDeletionPolicy1, HdfsDirectoryTest, TestElisionMultitermQuery, WrapperMergePolicyFactoryTest, TestQueryUtils, TestClusterProperties, TestHttpShardHandlerFactory, BasicDistributedZk2Test, TestRestoreCore, TestExtendedDismaxParser, BlobRepositoryCloudTest, TestNumericRangeQuery64, EnumFieldTest, MetricsConfigTest, ConcurrentDeleteAndCreateCollectionTest, OutOfBoxZkACLAndCredentialsProvidersTest, LeaderInitiatedRecoveryOnCommitTest, TestShardHandlerFactory, TestQueryTypes, JsonLoaderTest, SegmentsInfoRequestHandlerTest, DistributedDebugComponentTest, TestSolrFieldCacheBean, TestCollectionsAPIViaSolrCloudCluster, TestGroupingSearch, TestScoreJoinQPNoScore, TestConfigOverlay, TestLRUStatsCache, SynonymTokenizerTest, TestMaxScoreQueryParser, TestApiFramework, FullSolrCloudDistribCmdsTest, TestClusterStateMutator, MergeStrategyTest, TestManagedResource, SpellPossibilityIteratorTest, HdfsAutoAddReplicasIntegrationTest, OpenCloseCoreStressTest, SchemaVersionSpecificBehaviorTest, TestOverriddenPrefixQueryForCustomFieldType, TestCollationField, TestLockTree, TestPerFieldSimilarity, ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, SyncSliceTest, LeaderElectionIntegrationTest, ShardRoutingTest, BasicZkTest, RecoveryZkTest, ClusterStateUpdateTest, TestRandomFaceting, LeaderElectionTest, ShardRoutingCustomTest, TestDistributedSearch, TestDistributedGrouping, DistributedSpellCheckComponentTest, ZkControllerTest, TestRealTimeGet, TestStressReorder, TestJoin, DistributedTermsComponentTest, SolrCoreTest, StatsComponentTest, SolrCmdDistributorTest, QueryElevationComponentTest, PeerSyncTest, TestSort, TestFiltering, TestLazyCores, DirectUpdateHandlerTest, TestIndexSearcher, SoftAutoCommitTest, SimplePostToolTest, TestCoreDiscovery, AnalysisAfterCoreReloadTest, SuggesterFSTTest, TestFoldingMultitermQuery, DocValuesTest, SuggesterTest, TestStressLucene, WordBreakSolrSpellCheckerTest, TestPseudoReturnFields, FieldMutatingUpdateProcessorTest, TestWordDelimiterFilterFactory, QueryEqualityTest, DirectUpdateHandlerOptimizeTest, DefaultValueUpdateProcessorTest, SortByFunctionTest, TestRemoteStreaming, DistanceFunctionTest, SolrInfoBeanTest, CacheHeaderTest, TestSurroundQueryParser, DirectSolrSpellCheckerTest, PrimitiveFieldTypeTest, TestOmitPositions, FileBasedSpellCheckerTest, XmlUpdateRequestHandlerTest, TestValueSourceCache, TermVectorComponentTest, TestIndexingPerformance, IndexSchemaRuntimeFieldTest, LoggingHandlerTest, RegexBoostProcessorTest, SolrPluginUtilsTest, TestJmxIntegration, ReturnFieldsTest, TestCSVResponseWriter, MBeansHandlerTest, TestAnalyzedSuggestions, BinaryUpdateRequestHandlerTest, PingRequestHandlerTest, TestComponentsName, SearchHandlerTest, HighlighterConfigTest, TestQuerySenderListener, AlternateDirectoryTest, CopyFieldTest, SampleTest, TestDocSet, TestSearchPerf, MinimalSchemaTest, TestConfig, TestLuceneMatchVersion, TestXIncludeConfig, TestDFRSimilarityFactory, TestBM25SimilarityFactory, TimeZoneUtilsTest, ScriptEngineTest, URLClassifyProcessorTest, TestSuggestSpellingConverter, DOMUtilTest, TestSolrJ, TestLRUCache, ZkNodePropsTest, DistributedMLTComponentTest, TestLuceneIndexBackCompat, AliasIntegrationTest, AsyncCallRequestStatusResponseTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, ConfigSetsAPITest, ConnectionManagerTest, CreateCollectionCleanupTest, CustomCollectionTest]
[junit4] Completed [515/751 (2!)] on J1 in 109.89s, 4 tests, 3 errors <<< FAILURES!
[...truncated 48148 lines...]
[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 335 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/335/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC
2 tests failed.
FAILED: org.apache.solr.cloud.TestTlogReplica.testBasicLeaderElection
Error Message:
Stack Trace:
java.lang.NullPointerException
at __randomizedtesting.SeedInfo.seed([2D9263726D074962:BB1AA53A859D0E98]:0)
at org.apache.solr.cloud.TestTlogReplica.getSolrRunner(TestTlogReplica.java:897)
at org.apache.solr.cloud.TestTlogReplica.testBasicLeaderElection(TestTlogReplica.java:644)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestTlogReplica.testRecovery
Error Message:
Stack Trace:
java.lang.NullPointerException
at __randomizedtesting.SeedInfo.seed([2D9263726D074962:EC621ADE405783C5]:0)
at org.apache.solr.cloud.TestTlogReplica.getSolrRunner(TestTlogReplica.java:897)
at org.apache.solr.cloud.TestTlogReplica.testRecovery(TestTlogReplica.java:533)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13345 lines...]
[junit4] Suite: org.apache.solr.cloud.TestTlogReplica
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_2D9263726D074962-001/init-core-data-001
[junit4] 2> 2922836 WARN (SUITE-TestTlogReplica-seed#[2D9263726D074962]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=146 numCloses=146
[junit4] 2> 2922836 INFO (SUITE-TestTlogReplica-seed#[2D9263726D074962]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 2922839 INFO (SUITE-TestTlogReplica-seed#[2D9263726D074962]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 2922839 INFO (SUITE-TestTlogReplica-seed#[2D9263726D074962]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 2922840 INFO (SUITE-TestTlogReplica-seed#[2D9263726D074962]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_2D9263726D074962-001/tempDir-001
[junit4] 2> 2922840 INFO (SUITE-TestTlogReplica-seed#[2D9263726D074962]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2922841 INFO (Thread-4139) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2922841 INFO (Thread-4139) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2922843 ERROR (Thread-4139) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2922941 INFO (SUITE-TestTlogReplica-seed#[2D9263726D074962]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:55252
[junit4] 2> 2922958 INFO (zkConnectionManagerCallback-4577-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2922963 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160415a2f050000, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2922968 INFO (jetty-launcher-4574-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2922970 INFO (jetty-launcher-4574-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2922972 INFO (jetty-launcher-4574-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@b55d970{/solr,null,AVAILABLE}
[junit4] 2> 2922972 INFO (jetty-launcher-4574-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@36a7f19a{/solr,null,AVAILABLE}
[junit4] 2> 2922973 INFO (jetty-launcher-4574-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@286d00cc{HTTP/1.1,[http/1.1]}{127.0.0.1:55288}
[junit4] 2> 2922973 INFO (jetty-launcher-4574-thread-1) [ ] o.e.j.s.Server Started @2928008ms
[junit4] 2> 2922975 INFO (jetty-launcher-4574-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4e11d5d3{HTTP/1.1,[http/1.1]}{127.0.0.1:55289}
[junit4] 2> 2922975 INFO (jetty-launcher-4574-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=55288}
[junit4] 2> 2922975 INFO (jetty-launcher-4574-thread-2) [ ] o.e.j.s.Server Started @2928010ms
[junit4] 2> 2922975 INFO (jetty-launcher-4574-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=55289}
[junit4] 2> 2922975 ERROR (jetty-launcher-4574-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2922975 ERROR (jetty-launcher-4574-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2922975 INFO (jetty-launcher-4574-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 2922975 INFO (jetty-launcher-4574-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2922975 INFO (jetty-launcher-4574-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 2922975 INFO (jetty-launcher-4574-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2922976 INFO (jetty-launcher-4574-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2922976 INFO (jetty-launcher-4574-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-10T16:55:17.643Z
[junit4] 2> 2922976 INFO (jetty-launcher-4574-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2922976 INFO (jetty-launcher-4574-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-10T16:55:17.643Z
[junit4] 2> 2922977 INFO (zkConnectionManagerCallback-4580-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2922978 INFO (zkConnectionManagerCallback-4581-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2922978 INFO (jetty-launcher-4574-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2922979 INFO (jetty-launcher-4574-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2922988 INFO (jetty-launcher-4574-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55252/solr
[junit4] 2> 2922989 INFO (jetty-launcher-4574-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55252/solr
[junit4] 2> 2922990 INFO (zkConnectionManagerCallback-4587-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2922990 INFO (zkConnectionManagerCallback-4589-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2922991 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160415a2f050003, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2922992 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160415a2f050004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2922993 INFO (zkConnectionManagerCallback-4591-thread-1-processing-n:127.0.0.1:55289_solr) [n:127.0.0.1:55289_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2922993 INFO (zkConnectionManagerCallback-4593-thread-1-processing-n:127.0.0.1:55288_solr) [n:127.0.0.1:55288_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2923149 INFO (jetty-launcher-4574-thread-2) [n:127.0.0.1:55289_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2923149 INFO (jetty-launcher-4574-thread-1) [n:127.0.0.1:55288_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2923150 INFO (jetty-launcher-4574-thread-2) [n:127.0.0.1:55289_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:55289_solr
[junit4] 2> 2923151 INFO (jetty-launcher-4574-thread-1) [n:127.0.0.1:55288_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55288_solr
[junit4] 2> 2923152 INFO (jetty-launcher-4574-thread-2) [n:127.0.0.1:55289_solr ] o.a.s.c.Overseer Overseer (id=99151047393869829-127.0.0.1:55289_solr-n_0000000000) starting
[junit4] 2> 2923155 INFO (zkCallback-4590-thread-1-processing-n:127.0.0.1:55289_solr) [n:127.0.0.1:55289_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2923158 INFO (zkCallback-4592-thread-1-processing-n:127.0.0.1:55288_solr) [n:127.0.0.1:55288_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2923187 INFO (jetty-launcher-4574-thread-2) [n:127.0.0.1:55289_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55289_solr
[junit4] 2> 2923190 INFO (zkCallback-4590-thread-1-processing-n:127.0.0.1:55289_solr) [n:127.0.0.1:55289_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2923190 INFO (zkCallback-4592-thread-1-processing-n:127.0.0.1:55288_solr) [n:127.0.0.1:55288_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2924768 INFO (jetty-launcher-4574-thread-2) [n:127.0.0.1:55289_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55289.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@44bf27df
[junit4] 2> 2924769 INFO (jetty-launcher-4574-thread-1) [n:127.0.0.1:55288_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55288.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@44bf27df
[junit4] 2> 2924775 INFO (jetty-launcher-4574-thread-2) [n:127.0.0.1:55289_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55289.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@44bf27df
[junit4] 2> 2924775 INFO (jetty-launcher-4574-thread-2) [n:127.0.0.1:55289_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55289.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@44bf27df
[junit4] 2> 2924777 INFO (jetty-launcher-4574-thread-1) [n:127.0.0.1:55288_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55288.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@44bf27df
[junit4] 2> 2924777 INFO (jetty-launcher-4574-thread-2) [n:127.0.0.1:55289_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_2D9263726D074962-001/tempDir-001/node2/.
[junit4] 2> 2924777 INFO (jetty-launcher-4574-thread-1) [n:127.0.0.1:55288_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55288.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@44bf27df
[junit4] 2> 2924777 INFO (jetty-launcher-4574-thread-1) [n:127.0.0.1:55288_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_2D9263726D074962-001/tempDir-001/node1/.
[junit4] 2> 2924820 INFO (zkConnectionManagerCallback-4598-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2924824 INFO (zkConnectionManagerCallback-4602-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2924826 INFO (SUITE-TestTlogReplica-seed#[2D9263726D074962]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2924828 INFO (SUITE-TestTlogReplica-seed#[2D9263726D074962]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55252/solr ready
[junit4] 2> 2924833 INFO (SUITE-TestTlogReplica-seed#[2D9263726D074962]-worker) [ ] o.a.s.c.TestTlogReplica Using legacyCloud?: false
[junit4] 2> 2924835 INFO (qtp54455466-24136) [n:127.0.0.1:55289_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2924835 INFO (qtp54455466-24136) [n:127.0.0.1:55289_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2924989 INFO (TEST-TestTlogReplica.testOnlyLeaderIndexes-seed#[2D9263726D074962]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testOnlyLeaderIndexes
[junit4] 2> 2924991 INFO (qtp54455466-24137) [n:127.0.0.1:55289_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=0&collection.configName=conf&maxShardsPerNode=100&name=tlog_replica_test_only_leader_indexes&nrtReplicas=0&action=CREATE&numShards=1&tlogReplicas=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2924994 INFO (OverseerThreadFactory-9321-thread-1-processing-n:127.0.0.1:55289_solr) [n:127.0.0.1:55289_solr ] o.a.s.c.CreateCollectionCmd Create collection tlog_replica_test_only_leader_indexes
[junit4] 2> 2925116 INFO (OverseerStateUpdate-99151047393869829-127.0.0.1:55289_solr-n_0000000000) [n:127.0.0.1:55289_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"tlog_replica_test_only_leader_indexes",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"tlog_replica_test_only_leader_indexes_shard1_replica_t1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:55288/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2925122 INFO (OverseerStateUpdate-99151047393869829-127.0.0.1:55289_solr-n_0000000000) [n:127.0.0.1:55289_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"tlog_replica_test_only_leader_indexes",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"tlog_replica_test_only_leader_indexes_shard1_replica_t2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:55289/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2925349 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=tlog_replica_test_only_leader_indexes_shard1_replica_t2&action=CREATE&numShards=1&collection=tlog_replica_test_only_leader_indexes&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 2925349 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=tlog_replica_test_only_leader_indexes_shard1_replica_t1&action=CREATE&numShards=1&collection=tlog_replica_test_only_leader_indexes&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 2925350 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2925350 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2925473 INFO (zkCallback-4590-thread-1-processing-n:127.0.0.1:55289_solr) [n:127.0.0.1:55289_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_only_leader_indexes/state.json] for collection [tlog_replica_test_only_leader_indexes] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2925473 INFO (zkCallback-4592-thread-1-processing-n:127.0.0.1:55288_solr) [n:127.0.0.1:55288_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_only_leader_indexes/state.json] for collection [tlog_replica_test_only_leader_indexes] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2925474 INFO (zkCallback-4590-thread-2-processing-n:127.0.0.1:55289_solr) [n:127.0.0.1:55289_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_only_leader_indexes/state.json] for collection [tlog_replica_test_only_leader_indexes] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2925474 INFO (zkCallback-4592-thread-2-processing-n:127.0.0.1:55288_solr) [n:127.0.0.1:55288_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_only_leader_indexes/state.json] for collection [tlog_replica_test_only_leader_indexes] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2926373 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2926373 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2926379 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.s.IndexSchema [tlog_replica_test_only_leader_indexes_shard1_replica_t2] Schema name=minimal
[junit4] 2> 2926379 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.s.IndexSchema [tlog_replica_test_only_leader_indexes_shard1_replica_t1] Schema name=minimal
[junit4] 2> 2926383 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2926383 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_only_leader_indexes_shard1_replica_t1' using configuration from collection tlog_replica_test_only_leader_indexes, trusted=true
[junit4] 2> 2926383 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2926383 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_only_leader_indexes_shard1_replica_t2' using configuration from collection tlog_replica_test_only_leader_indexes, trusted=true
[junit4] 2> 2926383 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55288.solr.core.tlog_replica_test_only_leader_indexes.shard1.replica_t1' (registry 'solr.core.tlog_replica_test_only_leader_indexes.shard1.replica_t1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@44bf27df
[junit4] 2> 2926383 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55289.solr.core.tlog_replica_test_only_leader_indexes.shard1.replica_t2' (registry 'solr.core.tlog_replica_test_only_leader_indexes.shard1.replica_t2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@44bf27df
[junit4] 2> 2926384 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2926384 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2926384 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.SolrCore [[tlog_replica_test_only_leader_indexes_shard1_replica_t1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_2D9263726D074962-001/tempDir-001/node1/tlog_replica_test_only_leader_indexes_shard1_replica_t1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_2D9263726D074962-001/tempDir-001/node1/./tlog_replica_test_only_leader_indexes_shard1_replica_t1/data/]
[junit4] 2> 2926384 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.SolrCore [[tlog_replica_test_only_leader_indexes_shard1_replica_t2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_2D9263726D074962-001/tempDir-001/node2/tlog_replica_test_only_leader_indexes_shard1_replica_t2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_2D9263726D074962-001/tempDir-001/node2/./tlog_replica_test_only_leader_indexes_shard1_replica_t2/data/]
[junit4] 2> 2926526 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2926526 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2926556 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2926556 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2926578 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4cf857a4[tlog_replica_test_only_leader_indexes_shard1_replica_t1] main]
[junit4] 2> 2926580 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2926580 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2926581 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2926581 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1586416762222542848
[junit4] 2> 2926581 INFO (searcherExecutor-9326-thread-1-processing-n:127.0.0.1:55288_solr x:tlog_replica_test_only_leader_indexes_shard1_replica_t1 s:shard1 c:tlog_replica_test_only_leader_indexes r:core_node3) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.SolrCore [tlog_replica_test_only_leader_indexes_shard1_replica_t1] Registered new searcher Searcher@4cf857a4[tlog_replica_test_only_leader_indexes_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2926582 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2926582 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2926585 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2926585 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2926588 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 2926611 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@77b68007[tlog_replica_test_only_leader_indexes_shard1_replica_t2] main]
[junit4] 2> 2926613 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2926613 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2926614 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2926614 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1586416762257145856
[junit4] 2> 2926614 INFO (searcherExecutor-9327-thread-1-processing-n:127.0.0.1:55289_solr x:tlog_replica_test_only_leader_indexes_shard1_replica_t2 s:shard1 c:tlog_replica_test_only_leader_indexes r:core_node4) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.SolrCore [tlog_replica_test_only_leader_indexes_shard1_replica_t2] Registered new searcher Searcher@77b68007[tlog_replica_test_only_leader_indexes_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2926695 INFO (zkCallback-4590-thread-2-processing-n:127.0.0.1:55289_solr) [n:127.0.0.1:55289_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_only_leader_indexes/state.json] for collection [tlog_replica_test_only_leader_indexes] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2926695 INFO (zkCallback-4590-thread-3-processing-n:127.0.0.1:55289_solr) [n:127.0.0.1:55289_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_only_leader_indexes/state.json] for collection [tlog_replica_test_only_leader_indexes] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2926695 INFO (zkCallback-4592-thread-1-processing-n:127.0.0.1:55288_solr) [n:127.0.0.1:55288_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_only_leader_indexes/state.json] for collection [tlog_replica_test_only_leader_indexes] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2926695 INFO (zkCallback-4592-thread-2-processing-n:127.0.0.1:55288_solr) [n:127.0.0.1:55288_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_only_leader_indexes/state.json] for collection [tlog_replica_test_only_leader_indexes] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2927089 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2927089 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2927089 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:55288/solr/tlog_replica_test_only_leader_indexes_shard1_replica_t1/
[junit4] 2> 2927089 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_only_leader_indexes_shard1_replica_t1 url=http://127.0.0.1:55288/solr START replicas=[http://127.0.0.1:55289/solr/tlog_replica_test_only_leader_indexes_shard1_replica_t2/] nUpdates=100
[junit4] 2> 2927090 INFO (qtp54455466-24141) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2927390 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_only_leader_indexes_shard1_replica_t1 url=http://127.0.0.1:55288/solr DONE. We have no versions. sync failed.
[junit4] 2> 2927390 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 2927390 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 2927390 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2927391 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.ZkController tlog_replica_test_only_leader_indexes_shard1_replica_t1 stopping background replication from leader
[junit4] 2> 2927396 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:55288/solr/tlog_replica_test_only_leader_indexes_shard1_replica_t1/ shard1
[junit4] 2> 2927503 INFO (zkCallback-4590-thread-3-processing-n:127.0.0.1:55289_solr) [n:127.0.0.1:55289_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_only_leader_indexes/state.json] for collection [tlog_replica_test_only_leader_indexes] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2927503 INFO (zkCallback-4590-thread-1-processing-n:127.0.0.1:55289_solr) [n:127.0.0.1:55289_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_only_leader_indexes/state.json] for collection [tlog_replica_test_only_leader_indexes] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2927503 INFO (zkCallback-4592-thread-1-processing-n:127.0.0.1:55288_solr) [n:127.0.0.1:55288_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_only_leader_indexes/state.json] for collection [tlog_replica_test_only_leader_indexes] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2927503 INFO (zkCallback-4592-thread-2-processing-n:127.0.0.1:55288_solr) [n:127.0.0.1:55288_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_only_leader_indexes/state.json] for collection [tlog_replica_test_only_leader_indexes] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2927562 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2927566 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=tlog_replica_test_only_leader_indexes_shard1_replica_t1&action=CREATE&numShards=1&collection=tlog_replica_test_only_leader_indexes&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2216
[junit4] 2> 2927621 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.ZkController tlog_replica_test_only_leader_indexes_shard1_replica_t2 starting background replication from leader
[junit4] 2> 2927621 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 2927626 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 2927626 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2927626 INFO (indexFetcher-9337-thread-1) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.h.IndexFetcher Replica core_node3 is leader but it's state is down, skipping replication
[junit4] 2> 2927630 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=tlog_replica_test_only_leader_indexes_shard1_replica_t2&action=CREATE&numShards=1&collection=tlog_replica_test_only_leader_indexes&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2281
[junit4] 2> 2927638 INFO (qtp54455466-24137) [n:127.0.0.1:55289_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2927735 INFO (zkCallback-4590-thread-1-processing-n:127.0.0.1:55289_solr) [n:127.0.0.1:55289_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_only_leader_indexes/state.json] for collection [tlog_replica_test_only_leader_indexes] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2927735 INFO (zkCallback-4590-thread-3-processing-n:127.0.0.1:55289_solr) [n:127.0.0.1:55289_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_only_leader_indexes/state.json] for collection [tlog_replica_test_only_leader_indexes] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2927735 INFO (zkCallback-4592-thread-1-processing-n:127.0.0.1:55288_solr) [n:127.0.0.1:55288_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_only_leader_indexes/state.json] for collection [tlog_replica_test_only_leader_indexes] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2927735 INFO (zkCallback-4592-thread-2-processing-n:127.0.0.1:55288_solr) [n:127.0.0.1:55288_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_only_leader_indexes/state.json] for collection [tlog_replica_test_only_leader_indexes] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2928646 INFO (qtp54455466-24137) [n:127.0.0.1:55289_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=0&collection.configName=conf&maxShardsPerNode=100&name=tlog_replica_test_only_leader_indexes&nrtReplicas=0&action=CREATE&numShards=1&tlogReplicas=2&wt=javabin&version=2} status=0 QTime=3655
[junit4] 2> 2928668 INFO (qtp54455466-24133) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:55288/solr/tlog_replica_test_only_leader_indexes_shard1_replica_t1/&wt=javabin&version=2}{add=[1 (1586416764401483776), 2 (1586416764404629504), 3 (1586416764404629505), 4 (1586416764404629506)]} 0 1
[junit4] 2> 2928669 INFO (qtp1600519415-24129) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1 (1586416764401483776), 2 (1586416764404629504), 3 (1586416764404629505), 4 (1586416764404629506)]} 0 10
[junit4] 2> 2928680 INFO (qtp1600519415-24130) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/get params={distrib=false&qt=/get&id=1&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2928681 INFO (qtp1600519415-24132) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/get params={distrib=false&qt=/get&id=2&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2928683 INFO (qtp1600519415-24125) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/get params={distrib=false&qt=/get&id=3&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2928685 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/get params={distrib=false&qt=/get&id=4&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2928689 INFO (qtp54455466-24136) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/get params={distrib=false&qt=/get&id=1&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2928690 INFO (qtp54455466-24140) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/get params={distrib=false&qt=/get&id=2&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2928691 INFO (qtp54455466-24141) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/get params={distrib=false&qt=/get&id=3&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2928692 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/get params={distrib=false&qt=/get&id=4&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2928695 INFO (qtp54455466-24137) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:55288/solr/tlog_replica_test_only_leader_indexes_shard1_replica_t1/&wt=javabin&version=2}{delete=[1 (-1586416764437135360)]} 0 0
[junit4] 2> 2928695 INFO (qtp1600519415-24128) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1 (-1586416764437135360)]} 0 2
[junit4] 2> 2928701 INFO (qtp1600519415-24131) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@274017f6[tlog_replica_test_only_leader_indexes_shard1_replica_t1] realtime]
[junit4] 2> 2928704 INFO (qtp1600519415-24131) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@30841972[tlog_replica_test_only_leader_indexes_shard1_replica_t1] realtime]
[junit4] 2> 2928705 INFO (qtp54455466-24138) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&_version_=-1586416764441329664&distrib.from=http://127.0.0.1:55288/solr/tlog_replica_test_only_leader_indexes_shard1_replica_t1/&wt=javabin&version=2}{deleteByQuery=id:2 (-1586416764441329664)} 0 0
[junit4] 2> 2928706 INFO (qtp1600519415-24131) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=id:2 (-1586416764441329664)} 0 9
[junit4] 2> 2928709 INFO (qtp54455466-24140) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/get params={distrib=false&qt=/get&id=2&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2928709 INFO (qtp54455466-24141) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/get params={distrib=false&qt=/get&id=3&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2928710 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/get params={distrib=false&qt=/get&id=4&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2928712 INFO (qtp54455466-24133) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:55288/solr/tlog_replica_test_only_leader_indexes_shard1_replica_t1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
[junit4] 2> 2928712 INFO (qtp1600519415-24125) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1586416764457058304,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2928712 INFO (qtp1600519415-24125) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@75e8afde commitCommandVersion:1586416764457058304
[junit4] 2> 2928715 INFO (qtp1600519415-24125) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@56a15903[tlog_replica_test_only_leader_indexes_shard1_replica_t1] main]
[junit4] 2> 2928715 INFO (qtp1600519415-24125) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2928716 INFO (searcherExecutor-9326-thread-1-processing-n:127.0.0.1:55288_solr x:tlog_replica_test_only_leader_indexes_shard1_replica_t1 s:shard1 c:tlog_replica_test_only_leader_indexes r:core_node3) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.SolrCore [tlog_replica_test_only_leader_indexes_shard1_replica_t1] Registered new searcher Searcher@56a15903[tlog_replica_test_only_leader_indexes_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.3.0):C4/2:delGen=2)))}
[junit4] 2> 2928716 INFO (qtp1600519415-24125) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:55288/solr/tlog_replica_test_only_leader_indexes_shard1_replica_t1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 4
[junit4] 2> 2928717 INFO (qtp1600519415-24130) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/update params={_stateVer_=tlog_replica_test_only_leader_indexes:5&commit=true&wt=javabin&version=2}{commit=} 0 5
[junit4] 2> 2928719 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=2 status=0 QTime=0
[junit4] 2> 2928721 INFO (qtp54455466-24138) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2928828 INFO (qtp54455466-24136) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2928931 INFO (qtp54455466-24140) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2929002 INFO (OverseerCollectionConfigSetProcessor-99151047393869829-127.0.0.1:55289_solr-n_0000000000) [n:127.0.0.1:55289_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2929032 INFO (qtp54455466-24141) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2929135 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2929243 INFO (qtp54455466-24137) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2929344 INFO (qtp54455466-24133) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2929445 INFO (qtp54455466-24138) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2929553 INFO (qtp54455466-24136) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2929655 INFO (qtp54455466-24140) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2929766 INFO (qtp54455466-24141) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2929871 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2929981 INFO (qtp54455466-24137) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2930092 INFO (qtp54455466-24133) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2930199 INFO (qtp54455466-24138) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2930310 INFO (qtp54455466-24136) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2930420 INFO (qtp54455466-24140) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2930531 INFO (qtp54455466-24141) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2930636 INFO (indexFetcher-9337-thread-1) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication
[junit4] 2> 2930636 INFO (indexFetcher-9337-thread-1) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:55288/solr/tlog_replica_test_only_leader_indexes_shard1_replica_t1/
[junit4] 2> 2930637 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2930637 INFO (qtp1600519415-24132) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 2930638 INFO (indexFetcher-9337-thread-1) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.h.IndexFetcher Master's generation: 2
[junit4] 2> 2930638 INFO (indexFetcher-9337-thread-1) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.h.IndexFetcher Master's version: 1512924923379
[junit4] 2> 2930638 INFO (indexFetcher-9337-thread-1) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 2930638 INFO (indexFetcher-9337-thread-1) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 2930638 INFO (indexFetcher-9337-thread-1) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.h.IndexFetcher Starting replication process
[junit4] 2> 2930639 INFO (qtp1600519415-24125) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={generation=2&qt=/replication&tlogFiles=false&wt=javabin&version=2&command=filelist} status=0 QTime=0
[junit4] 2> 2930640 INFO (indexFetcher-9337-thread-1) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.h.IndexFetcher Number of files in latest index in master: 16
[junit4] 2> 2930640 INFO (indexFetcher-9337-thread-1) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.h.IndexFetcher Starting download (fullCopy=true) to MockDirectoryWrapper(RAMDirectory@22fe769d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ed7aaf4)
[junit4] 2> 2930641 INFO (qtp1600519415-24130) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_2.liv&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 2930643 INFO (qtp1600519415-24129) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 2930648 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Lucene50_0.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 2930649 INFO (qtp1600519415-24131) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Lucene50_0.tim&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 2930650 INFO (qtp1600519415-24128) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Lucene50_0.pos&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 2930652 INFO (qtp1600519415-24132) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.nvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 2930653 INFO (qtp1600519415-24125) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 2930655 INFO (qtp1600519415-24130) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Lucene50_0.tip&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 2930656 INFO (qtp1600519415-24129) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fdt&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 2930658 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Lucene70_0.dvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 2930659 INFO (qtp1600519415-24131) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Asserting_0.dvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 2930660 INFO (qtp1600519415-24128) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Lucene70_0.dvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 2930662 INFO (qtp1600519415-24132) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Asserting_0.dvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 2930663 INFO (qtp1600519415-24125) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.nvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 2930665 INFO (qtp1600519415-24130) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fnm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 2930666 INFO (qtp1600519415-24129) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/replication params={generation=2&qt=/replication&file=segments_2&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 2930667 INFO (indexFetcher-9337-thread-1) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.h.IndexFetcher Total time taken for download (fullCopy=true,bytesDownloaded=2540) : 0 secs (null bytes/sec) to MockDirectoryWrapper(RAMDirectory@22fe769d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ed7aaf4)
[junit4] 2> 2930667 INFO (indexFetcher-9337-thread-1) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.SolrCore Updating index properties... index=index.20171211035525307
[junit4] 2> 2930667 INFO (indexFetcher-9337-thread-1) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.h.IndexFetcher removing old index directory MockDirectoryWrapper(RAMDirectory@1a246d03 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55d8a759)
[junit4] 2> 2930668 INFO (indexFetcher-9337-thread-1) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 2930669 INFO (indexFetcher-9337-thread-1) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@4711fb2f[tlog_replica_test_only_leader_indexes_shard1_replica_t2] main]
[junit4] 2> 2930672 INFO (searcherExecutor-9327-thread-1-processing-n:127.0.0.1:55289_solr x:tlog_replica_test_only_leader_indexes_shard1_replica_t2 s:shard1 c:tlog_replica_test_only_leader_indexes r:core_node4) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.SolrCore [tlog_replica_test_only_leader_indexes_shard1_replica_t2] Registered new searcher Searcher@4711fb2f[tlog_replica_test_only_leader_indexes_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.3.0):C4/2:delGen=2)))}
[junit4] 2> 2930739 INFO (qtp54455466-24137) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=2 status=0 QTime=0
[junit4] 2> 2930750 INFO (qtp54455466-24133) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:55288/solr/tlog_replica_test_only_leader_indexes_shard1_replica_t1/&wt=javabin&version=2}{add=[15 (1586416766584619008)]} 0 1
[junit4] 2> 2930751 INFO (qtp1600519415-24127) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[15 (1586416766584619008)]} 0 10
[junit4] 2> 2930753 INFO (qtp54455466-24136) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:55288/solr/tlog_replica_test_only_leader_indexes_shard1_replica_t1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
[junit4] 2> 2930753 INFO (qtp1600519415-24128) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1586416766597201920,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2930753 INFO (qtp1600519415-24128) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@75e8afde commitCommandVersion:1586416766597201920
[junit4] 2> 2930756 INFO (qtp1600519415-24128) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@392f09e3[tlog_replica_test_only_leader_indexes_shard1_replica_t1] main]
[junit4] 2> 2930757 INFO (qtp1600519415-24128) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2930759 INFO (searcherExecutor-9326-thread-1-processing-n:127.0.0.1:55288_solr x:tlog_replica_test_only_leader_indexes_shard1_replica_t1 s:shard1 c:tlog_replica_test_only_leader_indexes r:core_node3) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.c.SolrCore [tlog_replica_test_only_leader_indexes_shard1_replica_t1] Registered new searcher Searcher@392f09e3[tlog_replica_test_only_leader_indexes_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.3.0):C4/2:delGen=2) Uninverting(_1(7.3.0):C1)))}
[junit4] 2> 2930759 INFO (qtp1600519415-24128) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:55288/solr/tlog_replica_test_only_leader_indexes_shard1_replica_t1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 5
[junit4] 2> 2930759 INFO (qtp1600519415-24131) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/update params={_stateVer_=tlog_replica_test_only_leader_indexes:5&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 7
[junit4] 2> 2930763 INFO (qtp54455466-24140) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:55288/solr/tlog_replica_test_only_leader_indexes_shard1_replica_t1/&wt=javabin&version=2}{add=[16 (1586416766605590528)]} 0 0
[junit4] 2> 2930764 INFO (qtp1600519415-24132) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[16 (1586416766605590528)]} 0 3
[junit4] 2> 2930766 INFO (qtp54455466-24141) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:55288/solr/tlog_replica_test_only_leader_indexes_shard1_replica_t1/&wt=javabin&version=2}{add=[17 (1586416766609784832)]} 0 0
[junit4] 2> 2930767 INFO (qtp1600519415-24125) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[17 (1586416766609784832)]} 0 1
[junit4] 2> 2930769 INFO (qtp54455466-24139) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:55288/solr/tlog_replica_test_only_leader_indexes_shard1_replica_t1/&wt=javabin&version=2}{add=[18 (1586416766612930560)]} 0 0
[junit4] 2> 2930769 INFO (qtp1600519415-24130) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[18 (1586416766612930560)]} 0 1
[junit4] 2> 2930771 INFO (qtp54455466-24137) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:55288/solr/tlog_replica_test_only_leader_indexes_shard1_replica_t1/&wt=javabin&version=2}{add=[19 (1586416766615027712)]} 0 0
[junit4] 2> 2930772 INFO (qtp1600519415-24129) [n:127.0.0.1:55288_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node3 x:tlog_replica_test_only_leader_indexes_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[19 (1586416766615027712)]} 0 1
[junit4] 2> 2930774 INFO (qtp54455466-24138) [n:127.0.0.1:55289_solr c:tlog_replica_test_only_leader_indexes s:shard1 r:core_node4 x:tlog_replica_test_only_leader_indexes_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_only_leader_indexes_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:55
[...truncated too long message...]
ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3026684 INFO (SUITE-TestTlogReplica-seed#[2D9263726D074962]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:55252 55252
[junit4] 2> 3026705 INFO (Thread-4139) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:55252 55252
[junit4] 2> 3026921 WARN (Thread-4139) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 26 /solr/configs/conf
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 5 /solr/security.json
[junit4] 2> 3 /solr/clusterprops.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 32 /solr/collections/tlog_replica_test_create_delete/state.json
[junit4] 2> 28 /solr/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json
[junit4] 2> 28 /solr/collections/tlog_replica_test_kill_leader/state.json
[junit4] 2> 26 /solr/collections/tlog_replica_test_remove_leader/state.json
[junit4] 2> 26 /solr/collections/tlog_replica_test_kill_tlog_replica/state.json
[junit4] 2> 24 /solr/collections/tlog_replica_test_add_remove_tlog_replica/state.json
[junit4] 2> 23 /solr/collections/tlog_replica_test_real_time_get/state.json
[junit4] 2> 15 /solr/collections/tlog_replica_test_recovery/state.json
[junit4] 2> 15 /solr/collections/tlog_replica_test_basic_leader_election/state.json
[junit4] 2> 15 /solr/collections/tlog_replica_test_delete_by_id/state.json
[junit4] 2> 13 /solr/collections/tlog_replica_test_add_docs/state.json
[junit4] 2> 13 /solr/collections/tlog_replica_test_only_leader_indexes/state.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 3 /solr/autoscaling.json
[junit4] 2> 3 /solr/collections/tlog_replica_test_create_delete/leader_elect/shard2/election/99151047393869838-core_node13-n_0000000000
[junit4] 2> 3 /solr/collections/tlog_replica_test_create_delete/leader_elect/shard1/election/99151047393869841-core_node7-n_0000000000
[junit4] 2> 2 /solr/overseer_elect/election/99151047393869830-127.0.0.1:55288_solr-n_0000000001
[junit4] 2> 2 /solr/collections/tlog_replica_test_create_delete/leader_elect/shard2/election/99151047393869838-core_node16-n_0000000002
[junit4] 2> 2 /solr/collections/tlog_replica_test_create_delete/leader_elect/shard2/election/99151047393869841-core_node11-n_0000000001
[junit4] 2> 2 /solr/collections/tlog_replica_test_create_delete/leader_elect/shard2/election/99151047393869841-core_node15-n_0000000001
[junit4] 2> 2 /solr/collections/tlog_replica_test_create_delete/leader_elect/shard1/election/99151047393869838-core_node5-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/live_nodes
[junit4] 2> 6 /solr/collections
[junit4] 2> 3 /solr/overseer/queue
[junit4] 2> 3 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 3 /solr/overseer/collection-queue-work
[junit4] 2> 3 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_2D9263726D074962-001
[junit4] 2> Dec 10, 2017 4:57:01 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {foo=PostingsFormat(name=Direct), title_s=PostingsFormat(name=Memory), id=Lucene50(blocksize=128), foo_s=PostingsFormat(name=Direct)}, docValues:{_version_=DocValuesFormat(name=Lucene70), id=DocValuesFormat(name=Asserting), inplace_updatable_int=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=259, maxMBSortInHeap=5.569460657260515, sim=RandomSimilarity(queryNorm=true): {}, locale=en-SG, timezone=Asia/Sakhalin
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=158284112,total=518979584
[junit4] 2> NOTE: All tests run in this JVM: [TestLegacyNumericUtils, AnalysisErrorHandlingTest, SimpleMLTQParserTest, JsonLoaderTest, TestManagedSchemaThreadSafety, SolrPluginUtilsTest, UtilsToolTest, HdfsAutoAddReplicasIntegrationTest, SortSpecParsingTest, TestAnalyzedSuggestions, BasicFunctionalityTest, AnalysisAfterCoreReloadTest, TestBM25SimilarityFactory, DeleteStatusTest, BaseCdcrDistributedZkTest, DirectSolrSpellCheckerTest, TestSolrFieldCacheBean, TestConfigSetProperties, BitVectorTest, TestNRTOpen, PreAnalyzedFieldManagedSchemaCloudTest, BlockDirectoryTest, DateRangeFieldTest, TestJsonFacetRefinement, TestMergePolicyConfig, SOLR749Test, TestUtilizeNode, TestDefaultStatsCache, LeaderElectionIntegrationTest, SuggestComponentContextFilterQueryTest, TestStressVersions, TestPerFieldSimilarity, MetricsConfigTest, TestTrackingShardHandlerFactory, RegexBytesRefFilterTest, HdfsWriteToMultipleCollectionsTest, CoreAdminRequestStatusTest, TestLockTree, TestChildDocTransformer, DistributedSuggestComponentTest, SolrGraphiteReporterTest, DistribDocExpirationUpdateProcessorTest, TestRandomCollapseQParserPlugin, TestDistributedMissingSort, UUIDUpdateProcessorFallbackTest, TestSort, TestJettySolrRunner, CacheHeaderTest, TestRandomFaceting, ChaosMonkeyNothingIsSafeWithPullReplicasTest, ZkNodePropsTest, TestLegacyField, MoreLikeThisHandlerTest, TestPolicyCloud, TestClusterStateMutator, AnalyticsMergeStrategyTest, TestCryptoKeys, TestSchemaResource, StatelessScriptUpdateProcessorFactoryTest, TestCoreBackup, DocValuesMultiTest, TestNumericTerms32, RuleEngineTest, TestFastWriter, TestRTGBase, CloneFieldUpdateProcessorFactoryTest, TestLeaderInitiatedRecoveryThread, SynonymTokenizerTest, TestCustomDocTransformer, TestFieldCacheSortRandom, DistributedQueueTest, TestFastOutputStream, TestCloudNestedDocsSort, TestRestoreCore, TestComponentsName, ConjunctionSolrSpellCheckerTest, PingRequestHandlerTest, TestCharFilters, MoveReplicaHDFSTest, TestSolrCLIRunExample, TestUseDocValuesAsStored, PropertiesRequestHandlerTest, HLLUtilTest, FileBasedSpellCheckerTest, TestHighFrequencyDictionaryFactory, TestReload, UpdateParamsTest, OverseerRolesTest, TestAnalyzeInfixSuggestions, TestSolrJ, DocValuesNotIndexedTest, TestIndexSearcher, TestApiFramework, TestFoldingMultitermQuery, TestExactSharedStatsCache, TestStreamBody, TestLMJelinekMercerSimilarityFactory, TestEmbeddedSolrServerConstructors, TestPartialUpdateDeduplication, SolrIndexMetricsTest, DistributedIntervalFacetingTest, TestCollectionsAPIViaSolrCloudCluster, SolrCoreTest, CopyFieldTest, TestShortCircuitedRequests, NodeAddedTriggerTest, TestReplicationHandlerBackup, TestMultiWordSynonyms, TestCollationFieldDocValues, FileUtilsTest, TestClusterProperties, MissingSegmentRecoveryTest, TestPerFieldSimilarityWithDefaultOverride, TestStressCloudBlindAtomicUpdates, AddSchemaFieldsUpdateProcessorFactoryTest, TestCopyFieldCollectionResource, TestObjectReleaseTracker, TestConfigOverlay, TestCollationField, AliasIntegrationTest, BigEndianAscendingWordDeserializerTest, CdcrReplicationHandlerTest, TestQueryUtils, TestQueryTypes, ComputePlanActionTest, OutOfBoxZkACLAndCredentialsProvidersTest, DocumentAnalysisRequestHandlerTest, TestLRUCache, TestCollapseQParserPlugin, TestSolrQueryParser, LoggingHandlerTest, TestCloudRecovery, DistributedFacetPivotLongTailTest, TestSimpleTrackingShardHandler, TestSQLHandlerNonCloud, TestExclusionRuleCollectionAccess, DistributedFacetExistsSmallTest, TestMultiValuedNumericRangeQuery, LegacyCloudClusterPropTest, TestStressReorder, TestRetrieveFieldsOptimizer, V2ApiIntegrationTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, SystemInfoHandlerTest, TestCrossCoreJoin, TestSchemaVersionResource, HdfsBasicDistributedZkTest, SmileWriterTest, PreAnalyzedFieldTest, TestDistribDocBasedVersion, TestDistributedGrouping, SampleTest, SimpleCollectionCreateDeleteTest, TestPullReplica, ReplaceNodeTest, ShardSplitTest, TestRemoteStreaming, SimpleFacetsTest, TestDistributedStatsComponentCardinality, HighlighterMaxOffsetTest, TestOverriddenPrefixQueryForCustomFieldType, AtomicUpdatesTest, SSLMigrationTest, UpdateLogTest, ClassificationUpdateProcessorIntegrationTest, ZkSolrClientTest, HdfsChaosMonkeySafeLeaderTest, ChangedSchemaMergeTest, EchoParamsTest, TestUniqueKeyFieldResource, TestComplexPhraseQParserPlugin, TestSkipOverseerOperations, TestExceedMaxTermLength, SolrCloudReportersTest, TestWordDelimiterFilterFactory, TestFieldSortValues, TestRequestForwarding, DocValuesTest, RequestLoggingTest, TestManagedResource, TestFaceting, ZkFailoverTest, TestMinMaxOnMultiValuedField, DocumentBuilderTest, BasicDistributedZkTest, TestRecovery, TermVectorComponentDistributedTest, DistributedTermsComponentTest, StatsComponentTest, BadIndexSchemaTest, TestFunctionQuery, TestBadConfig, SoftAutoCommitTest, AssignTest, CollectionsAPIAsyncDistributedZkTest, DeleteInactiveReplicaTest, DeleteLastCustomShardedReplicaTest, DeleteShardTest, DistribCursorPagingTest, ForceLeaderTest, LeaderElectionContextKeyTest, LeaderFailoverAfterPartitionTest, LeaderInitiatedRecoveryOnShardRestartTest, MigrateRouteKeyTest, NodeMutatorTest, OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, OverseerStatusTest, OverseerTaskQueueTest, PeerSyncReplicationTest, RemoteQueryErrorTest, SharedFSAutoReplicaFailoverTest, SolrCloudExampleTest, TestConfigSetsAPIExclusivity, TestConfigSetsAPIZkFailure, TestDeleteCollectionOnDownNodes, TestLeaderElectionZkExpiry, TestLocalFSCloudBackupRestore, TestRandomFlRTGCloud, TestRebalanceLeaders, TestSegmentSorting, TestTlogReplica]
[junit4] Completed [562/751 (1!)] on J1 in 104.64s, 13 tests, 2 errors <<< FAILURES!
[...truncated 48056 lines...]
[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 334 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/334/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC
4 tests failed.
FAILED: org.apache.solr.cloud.MultiThreadedOCPTest.test
Error Message:
Task 3001 did not complete, final state: FAILED expected same:<COMPLETED> was not:<FAILED>
Stack Trace:
java.lang.AssertionError: Task 3001 did not complete, final state: FAILED expected same:<COMPLETED> was not:<FAILED>
at __randomizedtesting.SeedInfo.seed([762C1519BB36C436:FE782AC315CAA9CE]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotSame(Assert.java:641)
at org.junit.Assert.assertSame(Assert.java:580)
at org.apache.solr.cloud.MultiThreadedOCPTest.testDeduplicationOfSubmittedTasks(MultiThreadedOCPTest.java:227)
at org.apache.solr.cloud.MultiThreadedOCPTest.test(MultiThreadedOCPTest.java:65)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test
Error Message:
Stack Trace:
java.lang.NullPointerException
at __randomizedtesting.SeedInfo.seed([762C1519BB36C436:FE782AC315CAA9CE]:0)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.getNodeName(JettySolrRunner.java:345)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:415)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:333)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.RecoveryAfterSoftCommitTest
Error Message:
18 threads leaked from SUITE scope at org.apache.solr.cloud.RecoveryAfterSoftCommitTest: 1) Thread[id=22323, name=zkCallback-6907-thread-3, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=22297, name=qtp2014065112-22297-acceptor-0@7e33132b-ServerConnector@3354c964{HTTP/1.1,[http/1.1]}{127.0.0.1:63200}, state=RUNNABLE, group=TGRP-RecoveryAfterSoftCommitTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=22316, name=zkCallback-6907-thread-1-processing-n:127.0.0.1:63199__uar%2Fyr, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:303) at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:518) at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:437) at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:424) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:788) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56) at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348) at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$1(SolrZkClient.java:268) at org.apache.solr.common.cloud.SolrZkClient$1$$Lambda$258/1060655241.run(Unknown Source) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$118/418487151.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=22296, name=qtp2014065112-22296, state=RUNNABLE, group=TGRP-RecoveryAfterSoftCommitTest] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=22309, name=Connection evictor, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=22298, name=qtp2014065112-22298, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=22300, name=qtp2014065112-22300, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=22324, name=zkCallback-6907-thread-4, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=22313, name=TEST-RecoveryAfterSoftCommitTest.test-seed#[762C1519BB36C436]-SendThread(127.0.0.1:63174), state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 10) Thread[id=22301, name=qtp2014065112-22301, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=22322, name=zkCallback-6907-thread-2, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=22308, name=Connection evictor, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=22299, name=qtp2014065112-22299, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=22304, name=org.eclipse.jetty.server.session.HashSessionManager@6ca73090Timer, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=22302, name=qtp2014065112-22302, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=22314, name=TEST-RecoveryAfterSoftCommitTest.test-seed#[762C1519BB36C436]-EventThread, state=WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 17) Thread[id=22315, name=zkConnectionManagerCallback-6908-thread-1, state=WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=22303, name=qtp2014065112-22303, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 18 threads leaked from SUITE scope at org.apache.solr.cloud.RecoveryAfterSoftCommitTest:
1) Thread[id=22323, name=zkCallback-6907-thread-3, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=22297, name=qtp2014065112-22297-acceptor-0@7e33132b-ServerConnector@3354c964{HTTP/1.1,[http/1.1]}{127.0.0.1:63200}, state=RUNNABLE, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=22316, name=zkCallback-6907-thread-1-processing-n:127.0.0.1:63199__uar%2Fyr, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:303)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:518)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:437)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:424)
at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:788)
at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$1(SolrZkClient.java:268)
at org.apache.solr.common.cloud.SolrZkClient$1$$Lambda$258/1060655241.run(Unknown Source)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$118/418487151.run(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=22296, name=qtp2014065112-22296, state=RUNNABLE, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=22309, name=Connection evictor, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=22298, name=qtp2014065112-22298, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=22300, name=qtp2014065112-22300, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=22324, name=zkCallback-6907-thread-4, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=22313, name=TEST-RecoveryAfterSoftCommitTest.test-seed#[762C1519BB36C436]-SendThread(127.0.0.1:63174), state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
10) Thread[id=22301, name=qtp2014065112-22301, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=22322, name=zkCallback-6907-thread-2, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=22308, name=Connection evictor, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=22299, name=qtp2014065112-22299, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=22304, name=org.eclipse.jetty.server.session.HashSessionManager@6ca73090Timer, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=22302, name=qtp2014065112-22302, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
16) Thread[id=22314, name=TEST-RecoveryAfterSoftCommitTest.test-seed#[762C1519BB36C436]-EventThread, state=WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
17) Thread[id=22315, name=zkConnectionManagerCallback-6908-thread-1, state=WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
18) Thread[id=22303, name=qtp2014065112-22303, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([762C1519BB36C436]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.RecoveryAfterSoftCommitTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=22323, name=zkCallback-6907-thread-3, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=22324, name=zkCallback-6907-thread-4, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=22313, name=TEST-RecoveryAfterSoftCommitTest.test-seed#[762C1519BB36C436]-SendThread(127.0.0.1:63174), state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 4) Thread[id=22316, name=zkCallback-6907-thread-1, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=22296, name=qtp2014065112-22296, state=RUNNABLE, group=TGRP-RecoveryAfterSoftCommitTest] at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method) at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:203) at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:257) at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213) at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:116) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=22322, name=zkCallback-6907-thread-2, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=22304, name=org.eclipse.jetty.server.session.HashSessionManager@6ca73090Timer, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=22315, name=zkConnectionManagerCallback-6908-thread-1, state=WAITING, group=TGRP-RecoveryAfterSoftCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=22323, name=zkCallback-6907-thread-3, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=22324, name=zkCallback-6907-thread-4, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=22313, name=TEST-RecoveryAfterSoftCommitTest.test-seed#[762C1519BB36C436]-SendThread(127.0.0.1:63174), state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
4) Thread[id=22316, name=zkCallback-6907-thread-1, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=22296, name=qtp2014065112-22296, state=RUNNABLE, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)
at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:203)
at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:257)
at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:116)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=22322, name=zkCallback-6907-thread-2, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=22304, name=org.eclipse.jetty.server.session.HashSessionManager@6ca73090Timer, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=22315, name=zkConnectionManagerCallback-6908-thread-1, state=WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([762C1519BB36C436]:0)
Build Log:
[...truncated 13229 lines...]
[junit4] Suite: org.apache.solr.cloud.MultiThreadedOCPTest
[junit4] 2> 2193371 INFO (SUITE-MultiThreadedOCPTest-seed#[762C1519BB36C436]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_762C1519BB36C436-001/init-core-data-001
[junit4] 2> 2193371 WARN (SUITE-MultiThreadedOCPTest-seed#[762C1519BB36C436]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=12 numCloses=12
[junit4] 2> 2193372 INFO (SUITE-MultiThreadedOCPTest-seed#[762C1519BB36C436]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 2193375 INFO (SUITE-MultiThreadedOCPTest-seed#[762C1519BB36C436]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 2193375 INFO (SUITE-MultiThreadedOCPTest-seed#[762C1519BB36C436]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /macu/gf
[junit4] 2> 2193380 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2193381 INFO (Thread-3558) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2193381 INFO (Thread-3558) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2193385 ERROR (Thread-3558) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2193491 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.ZkTestServer start zk server on port:63020
[junit4] 2> 2193530 INFO (zkConnectionManagerCallback-6816-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2193536 INFO (zkConnectionManagerCallback-6818-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2193545 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 2193549 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 2193554 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 2193556 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 2193559 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 2193562 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 2193565 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 2193567 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 2193570 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 2193573 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 2193575 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 2193579 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 2193992 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2193994 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@622743e4{/macu/gf,null,AVAILABLE}
[junit4] 2> 2193998 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@f996515{HTTP/1.1,[http/1.1]}{127.0.0.1:63023}
[junit4] 2> 2193999 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.e.j.s.Server Started @2200419ms
[junit4] 2> 2193999 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_762C1519BB36C436-001/tempDir-001/control/data, replicaType=NRT, hostContext=/macu/gf, hostPort=63023, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_762C1519BB36C436-001/control-001/cores}
[junit4] 2> 2193999 ERROR (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2193999 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 2193999 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2193999 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2193999 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-10T05:47:51.832Z
[junit4] 2> 2194001 INFO (zkConnectionManagerCallback-6820-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2194002 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2194002 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_762C1519BB36C436-001/control-001/solr.xml
[junit4] 2> 2194009 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2194009 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2194010 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@71188b4d, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2194014 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:63020/solr
[junit4] 2> 2194016 INFO (zkConnectionManagerCallback-6824-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2194018 INFO (zkConnectionManagerCallback-6826-thread-1-processing-n:127.0.0.1:63023_macu%2Fgf) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2194160 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2194161 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:63023_macu%2Fgf
[junit4] 2> 2194162 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.Overseer Overseer (id=99148422919946244-127.0.0.1:63023_macu%2Fgf-n_0000000000) starting
[junit4] 2> 2194172 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63023_macu%2Fgf
[junit4] 2> 2194176 INFO (OverseerStateUpdate-99148422919946244-127.0.0.1:63023_macu%2Fgf-n_0000000000) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2194296 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71188b4d
[junit4] 2> 2194305 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71188b4d
[junit4] 2> 2194305 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71188b4d
[junit4] 2> 2194306 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_762C1519BB36C436-001/control-001/cores
[junit4] 2> 2194329 INFO (zkConnectionManagerCallback-6832-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2194330 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2194331 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:63020/solr ready
[junit4] 2> 2194333 INFO (qtp750190058-21886) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:63023_macu%252Fgf&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2194337 INFO (OverseerThreadFactory-8695-thread-1-processing-n:127.0.0.1:63023_macu%2Fgf) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 2194447 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2194448 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2194563 INFO (zkCallback-6825-thread-1-processing-n:127.0.0.1:63023_macu%2Fgf) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2195495 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2195509 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 2195583 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2195602 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 2195602 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71188b4d
[junit4] 2> 2195602 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2195602 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_762C1519BB36C436-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_762C1519BB36C436-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 2195604 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=17.6279296875, floorSegmentMB=0.9326171875, forceMergeDeletesPctAllowed=3.098247802283808, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5289119945256479
[junit4] 2> 2195608 WARN (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2195648 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2195648 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2195649 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2195649 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2195650 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=14, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 2195650 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@53065f0d[control_collection_shard1_replica_n1] main]
[junit4] 2> 2195652 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2195652 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2195653 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2195653 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1586374769100455936
[junit4] 2> 2195654 INFO (searcherExecutor-8698-thread-1-processing-n:127.0.0.1:63023_macu%2Fgf x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@53065f0d[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2195658 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2195658 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2195658 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:63023/macu/gf/control_collection_shard1_replica_n1/
[junit4] 2> 2195658 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2195658 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:63023/macu/gf/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 2195658 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2195663 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:63023/macu/gf/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 2195770 INFO (zkCallback-6825-thread-1-processing-n:127.0.0.1:63023_macu%2Fgf) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2195780 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2195784 INFO (qtp750190058-21887) [n:127.0.0.1:63023_macu%2Fgf c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1337
[junit4] 2> 2195796 INFO (qtp750190058-21886) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2195892 INFO (zkCallback-6825-thread-1-processing-n:127.0.0.1:63023_macu%2Fgf) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2196344 INFO (OverseerCollectionConfigSetProcessor-99148422919946244-127.0.0.1:63023_macu%2Fgf-n_0000000000) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2196799 INFO (qtp750190058-21886) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:63023_macu%252Fgf&wt=javabin&version=2} status=0 QTime=2465
[junit4] 2> 2196809 INFO (zkConnectionManagerCallback-6836-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2196812 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2196815 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:63020/solr ready
[junit4] 2> 2196816 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 2196817 INFO (qtp750190058-21891) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2196822 INFO (OverseerThreadFactory-8695-thread-2-processing-n:127.0.0.1:63023_macu%2Fgf) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 2196825 WARN (OverseerThreadFactory-8695-thread-2-processing-n:127.0.0.1:63023_macu%2Fgf) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 2197047 INFO (qtp750190058-21891) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2197048 INFO (qtp750190058-21891) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=230
[junit4] 2> 2197301 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_762C1519BB36C436-001/shard-1-001 of type NRT
[junit4] 2> 2197302 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2197302 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@28c431b1{/macu/gf,null,AVAILABLE}
[junit4] 2> 2197302 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4ad17bbf{HTTP/1.1,[http/1.1]}{127.0.0.1:63036}
[junit4] 2> 2197302 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.e.j.s.Server Started @2203722ms
[junit4] 2> 2197302 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_762C1519BB36C436-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/macu/gf, hostPort=63036, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_762C1519BB36C436-001/shard-1-001/cores}
[junit4] 2> 2197302 ERROR (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2197302 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 2197302 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2197302 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2197302 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-10T05:47:55.135Z
[junit4] 2> 2197307 INFO (zkConnectionManagerCallback-6838-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2197308 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2197308 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_762C1519BB36C436-001/shard-1-001/solr.xml
[junit4] 2> 2197312 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2197312 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2197312 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@71188b4d, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2197315 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:63020/solr
[junit4] 2> 2197316 INFO (zkConnectionManagerCallback-6842-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2197329 INFO (zkConnectionManagerCallback-6844-thread-1-processing-n:127.0.0.1:63036_macu%2Fgf) [n:127.0.0.1:63036_macu%2Fgf ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2197339 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [n:127.0.0.1:63036_macu%2Fgf ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2197343 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [n:127.0.0.1:63036_macu%2Fgf ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2197346 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [n:127.0.0.1:63036_macu%2Fgf ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63036_macu%2Fgf
[junit4] 2> 2197347 INFO (zkCallback-6835-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2197348 INFO (zkCallback-6825-thread-1-processing-n:127.0.0.1:63023_macu%2Fgf) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2197350 INFO (zkCallback-6843-thread-1-processing-n:127.0.0.1:63036_macu%2Fgf) [n:127.0.0.1:63036_macu%2Fgf ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2197399 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [n:127.0.0.1:63036_macu%2Fgf ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71188b4d
[junit4] 2> 2197407 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [n:127.0.0.1:63036_macu%2Fgf ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71188b4d
[junit4] 2> 2197407 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [n:127.0.0.1:63036_macu%2Fgf ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71188b4d
[junit4] 2> 2197408 INFO (TEST-MultiThreadedOCPTest.test-seed#[762C1519BB36C436]) [n:127.0.0.1:63036_macu%2Fgf ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_762C1519BB36C436-001/shard-1-001/cores
[junit4] 2> 2197430 INFO (qtp750190058-21884) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:63036_macu%252Fgf&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2197433 INFO (OverseerCollectionConfigSetProcessor-99148422919946244-127.0.0.1:63023_macu%2Fgf-n_0000000000) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2197433 INFO (OverseerThreadFactory-8695-thread-3-processing-n:127.0.0.1:63023_macu%2Fgf) [n:127.0.0.1:63023_macu%2Fgf ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:63036_macu%2Fgf for creating new replica
[junit4] 2> 2197437 INFO (qtp1523290185-21935) [n:127.0.0.1:63036_macu%2Fgf ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2197437 INFO (qtp1523290185-21935) [n:127.0.0.1:63036_macu%2Fgf ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2198481 INFO (qtp1523290185-21935) [n:127.0.0.1:63036_macu%2Fgf c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2198495 INFO (qtp1523290185-21935) [n:127.0.0.1:63036_macu%2Fgf c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema [collection1_shard2_replica_n41] Schema name=test
[junit4] 2> 2198569 INFO (qtp1523290185-21935) [n:127.0.0.1:63036_macu%2Fgf c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2198588 INFO (qtp1523290185-21935) [n:127.0.0.1:63036_macu%2Fgf c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n41' using configuration from collection collection1, trusted=true
[junit4] 2> 2198588 INFO (qtp1523290185-21935) [n:127.0.0.1:63036_macu%2Fgf c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n41' (registry 'solr.core.collection1.shard2.replica_n41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@71188b4d
[junit4] 2> 2198588 INFO (qtp1523290185-21935) [n:127.0.0.1:63036_macu%2Fgf c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2198588 INFO (qtp1523290185-21935) [n:127.0.0.1:63036_macu%2Fgf c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [[collection1_shard2_replica_n41] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_762C1519BB36C436-001/shard-1-001/cores/collection1_shard2_replica_n41], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_762C1519BB36C436-001/shard-1-001/cores/collection1_shard2_replica_n41/data/]
[junit4] 2> 2198590 INFO (qtp1523290185-21935) [n:127.0.0.1:63036_macu%2Fgf c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=17.6279296875, floorSegmentMB=0.9326171875, forceMergeDeletesPctAllowed=3.098247802283808, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5289119945256479
[junit4] 2> 2198592 WARN (qtp1523290185-21935) [n:127.0.0.1:63036_macu%2Fgf c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2198631 INFO (qtp1523290185-21935) [n:127.0.0.1:63036_macu%2Fgf c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[juni
[...truncated too long message...]
or.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 18) Thread[id=22303, name=qtp2014065112-22303, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([762C1519BB36C436]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=22323, name=zkCallback-6907-thread-3, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 2) Thread[id=22324, name=zkCallback-6907-thread-4, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 3) Thread[id=22313, name=TEST-RecoveryAfterSoftCommitTest.test-seed#[762C1519BB36C436]-SendThread(127.0.0.1:63174), state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
[junit4] > 4) Thread[id=22316, name=zkCallback-6907-thread-1, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=22296, name=qtp2014065112-22296, state=RUNNABLE, group=TGRP-RecoveryAfterSoftCommitTest]
[junit4] > at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)
[junit4] > at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:203)
[junit4] > at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:257)
[junit4] > at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
[junit4] > at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
[junit4] > at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:116)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=22322, name=zkCallback-6907-thread-2, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 7) Thread[id=22304, name=org.eclipse.jetty.server.session.HashSessionManager@6ca73090Timer, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 8) Thread[id=22315, name=zkConnectionManagerCallback-6908-thread-1, state=WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([762C1519BB36C436]:0)
[junit4] Completed [536/751 (2!)] on J0 in 28.23s, 1 test, 3 errors <<< FAILURES!
[...truncated 48093 lines...]