You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/03/14 20:36:52 UTC
[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 760 -
Still Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/760/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
6 tests failed.
FAILED: org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForImplicitRouter
Error Message:
Collection not found: withShardField
Stack Trace:
org.apache.solr.common.SolrException: Collection not found: withShardField
at __randomizedtesting.SeedInfo.seed([87434765DE425C2C:D213AFF772BB93DC]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionNames(CloudSolrClient.java:1394)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1087)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1057)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:160)
at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:232)
at org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForImplicitRouter(CustomCollectionTest.java:141)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
FAILED: org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI
Error Message:
Could not find collection : implicitcoll
Stack Trace:
org.apache.solr.common.SolrException: Could not find collection : implicitcoll
at __randomizedtesting.SeedInfo.seed([87434765DE425C2C:EDA2C90EE3D8EA54]:0)
at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:194)
at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:245)
at org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI(CustomCollectionTest.java:68)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
FAILED: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test
Error Message:
java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
Stack Trace:
org.apache.solr.common.SolrException: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:170)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:111)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:98)
at org.apache.solr.cloud.AbstractDistribZkTestBase.printLayout(AbstractDistribZkTestBase.java:322)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:668)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1782)
at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:447)
at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380)
at org.apache.solr.common.cloud.SolrZooKeeper.<init>(SolrZooKeeper.java:43)
at org.apache.solr.common.cloud.ZkClientConnectionStrategy.createSolrZooKeeper(ZkClientConnectionStrategy.java:105)
at org.apache.solr.common.cloud.DefaultConnectionStrategy.connect(DefaultConnectionStrategy.java:37)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:145)
... 47 more
Caused by: java.io.IOException: Too many open files
at sun.nio.ch.IOUtil.makePipe(Native Method)
at sun.nio.ch.KQueueSelectorImpl.<init>(KQueueSelectorImpl.java:84)
at sun.nio.ch.KQueueSelectorProvider.openSelector(KQueueSelectorProvider.java:42)
at java.nio.channels.Selector.open(Selector.java:227)
at org.apache.zookeeper.ClientCnxnSocketNIO.<init>(ClientCnxnSocketNIO.java:43)
at sun.reflect.GeneratedConstructorAccessor79.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at java.lang.Class.newInstance(Class.java:442)
at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1779)
... 53 more
FAILED: org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.testNoSslButSillyClientAuth
Error Message:
java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
Stack Trace:
org.apache.solr.common.SolrException: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
at __randomizedtesting.SeedInfo.seed([87434765DE425C2C:EA15F3DB3ACBFFC1]:0)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:170)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:111)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:98)
at org.apache.solr.cloud.MiniSolrCloudCluster.waitForAllNodes(MiniSolrCloudCluster.java:270)
at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:264)
at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:197)
at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:124)
at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.checkClusterWithNodeReplacement(TestMiniSolrCloudClusterSSL.java:135)
at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.testNoSslButSillyClientAuth(TestMiniSolrCloudClusterSSL.java:102)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1782)
at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:447)
at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380)
at org.apache.solr.common.cloud.SolrZooKeeper.<init>(SolrZooKeeper.java:43)
at org.apache.solr.common.cloud.ZkClientConnectionStrategy.createSolrZooKeeper(ZkClientConnectionStrategy.java:105)
at org.apache.solr.common.cloud.DefaultConnectionStrategy.connect(DefaultConnectionStrategy.java:37)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:145)
... 48 more
Caused by: java.io.IOException: Too many open files
at sun.nio.ch.IOUtil.makePipe(Native Method)
at sun.nio.ch.KQueueSelectorImpl.<init>(KQueueSelectorImpl.java:84)
at sun.nio.ch.KQueueSelectorProvider.openSelector(KQueueSelectorProvider.java:42)
at java.nio.channels.Selector.open(Selector.java:227)
at org.apache.zookeeper.ClientCnxnSocketNIO.<init>(ClientCnxnSocketNIO.java:43)
at sun.reflect.GeneratedConstructorAccessor79.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at java.lang.Class.newInstance(Class.java:442)
at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1779)
... 54 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestMiniSolrCloudClusterSSL
Error Message:
55 threads leaked from SUITE scope at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL: 1) Thread[id=23017, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Object.wait(Native Method) at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146) 2) Thread[id=23090, name=OverseerCollectionConfigSetProcessor-97617312122732551-127.0.0.1:60461_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:343) at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:227) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=23043, name=qtp909562513-23043, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=23041, name=qtp909562513-23041-acceptor-0@4521ebad-ServerConnector@705861b6{HTTP/1.1,[http/1.1]}{127.0.0.1:60463}, state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=23024, name=jetty-launcher-5355-thread-2, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=23025, name=jetty-launcher-5355-thread-3, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=23038, name=Scheduler-1312424807, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=23051, name=org.eclipse.jetty.server.session.HashSessionManager@6f4c21d2Timer, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=23080, name=jetty-launcher-5355-thread-3-EventThread, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 10) Thread[id=23019, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120) 11) Thread[id=23035, name=qtp909562513-23035, state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 12) Thread[id=23086, name=jetty-launcher-5355-thread-1-SendThread(127.0.0.1:60459), state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) 13) Thread[id=23039, name=qtp650869621-23039, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 14) Thread[id=23052, name=qtp909562513-23052, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 15) Thread[id=23030, name=qtp858609896-23030, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 16) Thread[id=23082, name=jetty-launcher-5355-thread-2-EventThread, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 17) Thread[id=23088, name=zkCallback-5372-thread-1, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 18) Thread[id=23031, name=qtp858609896-23031, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 19) Thread[id=23091, name=OverseerHdfsCoreFailoverThread-97617312122732551-127.0.0.1:60461_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139) at java.lang.Thread.run(Thread.java:745) 20) Thread[id=23042, name=qtp909562513-23042, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 21) Thread[id=23067, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:258) at java.lang.Thread.run(Thread.java:745) 22) Thread[id=23081, name=jetty-launcher-5355-thread-2-SendThread(127.0.0.1:60459), state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) 23) Thread[id=23079, name=jetty-launcher-5355-thread-3-SendThread(127.0.0.1:60459), state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) 24) Thread[id=23087, name=jetty-launcher-5355-thread-1-EventThread, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 25) Thread[id=23037, name=qtp650869621-23037, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 26) Thread[id=23053, name=qtp909562513-23053, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 27) Thread[id=23029, name=qtp858609896-23029, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 28) Thread[id=23076, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:258) at java.lang.Thread.run(Thread.java:745) 29) Thread[id=23049, name=Scheduler-212900710, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 30) Thread[id=23070, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:258) at java.lang.Thread.run(Thread.java:745) 31) Thread[id=23033, name=qtp650869621-23033, state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 32) Thread[id=23054, name=Scheduler-1910054328, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 33) Thread[id=23026, name=qtp858609896-23026, state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 34) Thread[id=23046, name=qtp650869621-23046, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 35) Thread[id=23016, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178) at java.lang.Thread.run(Thread.java:745) 36) Thread[id=23027, name=qtp858609896-23027-acceptor-0@6c9b15e4-ServerConnector@6c1ea61c{HTTP/1.1,[http/1.1]}{127.0.0.1:60462}, state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 37) Thread[id=23066, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:258) at java.lang.Thread.run(Thread.java:745) 38) Thread[id=23034, name=qtp858609896-23034, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 39) Thread[id=23085, name=zkCallback-5371-thread-1, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 40) Thread[id=23023, name=jetty-launcher-5355-thread-1, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 41) Thread[id=23032, name=qtp858609896-23032, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 42) Thread[id=23084, name=zkCallback-5370-thread-1, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 43) Thread[id=23045, name=qtp650869621-23045, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 44) Thread[id=23040, name=qtp650869621-23040, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 45) Thread[id=23036, name=qtp650869621-23036-acceptor-0@54db9d5c-ServerConnector@4d6cb4d7{HTTP/1.1,[http/1.1]}{127.0.0.1:60461}, state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 46) Thread[id=23047, name=org.eclipse.jetty.server.session.HashSessionManager@204e884bTimer, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 47) Thread[id=23075, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:258) at java.lang.Thread.run(Thread.java:745) 48) Thread[id=23028, name=qtp858609896-23028, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 49) Thread[id=23065, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:258) at java.lang.Thread.run(Thread.java:745) 50) Thread[id=23048, name=qtp650869621-23048, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 51) Thread[id=23050, name=qtp909562513-23050, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 52) Thread[id=23015, name=Thread-3684, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Object.wait(Native Method) at java.lang.Thread.join(Thread.java:1249) at java.lang.Thread.join(Thread.java:1323) at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302) at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309) at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490) 53) Thread[id=23044, name=qtp909562513-23044, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 54) Thread[id=23055, name=org.eclipse.jetty.server.session.HashSessionManager@30ea6542Timer, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 55) Thread[id=23089, name=OverseerStateUpdate-97617312122732551-127.0.0.1:60461_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:138) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:180) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 55 threads leaked from SUITE scope at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL:
1) Thread[id=23017, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at java.lang.Object.wait(Native Method)
at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
2) Thread[id=23090, name=OverseerCollectionConfigSetProcessor-97617312122732551-127.0.0.1:60461_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:343)
at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)
at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:227)
at java.lang.Thread.run(Thread.java:745)
3) Thread[id=23043, name=qtp909562513-23043, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
4) Thread[id=23041, name=qtp909562513-23041-acceptor-0@4521ebad-ServerConnector@705861b6{HTTP/1.1,[http/1.1]}{127.0.0.1:60463}, state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
5) Thread[id=23024, name=jetty-launcher-5355-thread-2, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
6) Thread[id=23025, name=jetty-launcher-5355-thread-3, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
7) Thread[id=23038, name=Scheduler-1312424807, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
8) Thread[id=23051, name=org.eclipse.jetty.server.session.HashSessionManager@6f4c21d2Timer, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
9) Thread[id=23080, name=jetty-launcher-5355-thread-3-EventThread, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
10) Thread[id=23019, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
11) Thread[id=23035, name=qtp909562513-23035, state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
12) Thread[id=23086, name=jetty-launcher-5355-thread-1-SendThread(127.0.0.1:60459), state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
13) Thread[id=23039, name=qtp650869621-23039, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
14) Thread[id=23052, name=qtp909562513-23052, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
15) Thread[id=23030, name=qtp858609896-23030, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
16) Thread[id=23082, name=jetty-launcher-5355-thread-2-EventThread, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
17) Thread[id=23088, name=zkCallback-5372-thread-1, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
18) Thread[id=23031, name=qtp858609896-23031, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
19) Thread[id=23091, name=OverseerHdfsCoreFailoverThread-97617312122732551-127.0.0.1:60461_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
at java.lang.Thread.run(Thread.java:745)
20) Thread[id=23042, name=qtp909562513-23042, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
21) Thread[id=23067, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:258)
at java.lang.Thread.run(Thread.java:745)
22) Thread[id=23081, name=jetty-launcher-5355-thread-2-SendThread(127.0.0.1:60459), state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
23) Thread[id=23079, name=jetty-launcher-5355-thread-3-SendThread(127.0.0.1:60459), state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
24) Thread[id=23087, name=jetty-launcher-5355-thread-1-EventThread, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
25) Thread[id=23037, name=qtp650869621-23037, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
26) Thread[id=23053, name=qtp909562513-23053, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
27) Thread[id=23029, name=qtp858609896-23029, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
28) Thread[id=23076, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:258)
at java.lang.Thread.run(Thread.java:745)
29) Thread[id=23049, name=Scheduler-212900710, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
30) Thread[id=23070, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:258)
at java.lang.Thread.run(Thread.java:745)
31) Thread[id=23033, name=qtp650869621-23033, state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
32) Thread[id=23054, name=Scheduler-1910054328, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
33) Thread[id=23026, name=qtp858609896-23026, state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
34) Thread[id=23046, name=qtp650869621-23046, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
35) Thread[id=23016, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
at java.lang.Thread.run(Thread.java:745)
36) Thread[id=23027, name=qtp858609896-23027-acceptor-0@6c9b15e4-ServerConnector@6c1ea61c{HTTP/1.1,[http/1.1]}{127
[...truncated too long message...]
name=jetty-launcher-5355-thread-2-SendThread(127.0.0.1:60459), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
[junit4] > 11) Thread[id=23026, name=qtp858609896-23026, state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)
[junit4] > at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:203)
[junit4] > at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:257)
[junit4] > at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
[junit4] > at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
[junit4] > at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:116)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 12) Thread[id=23038, name=Scheduler-1312424807, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 13) Thread[id=23079, name=jetty-launcher-5355-thread-3-SendThread(127.0.0.1:60459), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
[junit4] > 14) Thread[id=23051, name=org.eclipse.jetty.server.session.HashSessionManager@6f4c21d2Timer, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 15) Thread[id=23035, name=qtp909562513-23035, state=RUNNABLE, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)
[junit4] > at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:203)
[junit4] > at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:257)
[junit4] > at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
[junit4] > at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
[junit4] > at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:116)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 16) Thread[id=23055, name=org.eclipse.jetty.server.session.HashSessionManager@30ea6542Timer, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 17) Thread[id=23086, name=jetty-launcher-5355-thread-1-SendThread(127.0.0.1:60459), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
[junit4] > 18) Thread[id=23049, name=Scheduler-212900710, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([87434765DE425C2C]:0)
[junit4] Completed [520/698 (3!)] on J1 in 63.07s, 4 tests, 3 errors, 1 skipped <<< FAILURES!
[...truncated 63286 lines...]
[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 762 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/762/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
1 tests failed.
FAILED: org.apache.solr.cloud.MultiThreadedOCPTest.test
Error Message:
Task 3001 did not complete, final state: FAILED expected same:<COMPLETED> was not:<FAILED>
Stack Trace:
java.lang.AssertionError: Task 3001 did not complete, final state: FAILED expected same:<COMPLETED> was not:<FAILED>
at __randomizedtesting.SeedInfo.seed([AA2BD94A175182ED:227FE690B9ADEF15]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotSame(Assert.java:641)
at org.junit.Assert.assertSame(Assert.java:580)
at org.apache.solr.cloud.MultiThreadedOCPTest.testDeduplicationOfSubmittedTasks(MultiThreadedOCPTest.java:250)
at org.apache.solr.cloud.MultiThreadedOCPTest.test(MultiThreadedOCPTest.java:64)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12507 lines...]
[junit4] Suite: org.apache.solr.cloud.MultiThreadedOCPTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/init-core-data-001
[junit4] 2> 2600582 WARN (SUITE-MultiThreadedOCPTest-seed#[AA2BD94A175182ED]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=12 numCloses=12
[junit4] 2> 2600582 INFO (SUITE-MultiThreadedOCPTest-seed#[AA2BD94A175182ED]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields
[junit4] 2> 2600587 INFO (SUITE-MultiThreadedOCPTest-seed#[AA2BD94A175182ED]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 2600587 INFO (SUITE-MultiThreadedOCPTest-seed#[AA2BD94A175182ED]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 2600588 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2600589 INFO (Thread-3509) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2600589 INFO (Thread-3509) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2600699 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.ZkTestServer start zk server on port:56046
[junit4] 2> 2600730 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 2600734 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 2600737 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 2600739 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 2600743 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 2600745 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 2600748 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 2600751 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 2600754 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 2600757 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 2600759 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 2601035 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/control-001/cores/collection1
[junit4] 2> 2601038 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2601040 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3ec54f3d{/,null,AVAILABLE}
[junit4] 2> 2601041 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@506de133{HTTP/1.1,[http/1.1]}{127.0.0.1:56049}
[junit4] 2> 2601041 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.Server Started @2607826ms
[junit4] 2> 2601041 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/tempDir-001/control/data, hostContext=/, hostPort=56049, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/control-001/cores}
[junit4] 2> 2601042 ERROR (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2601042 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.5.0
[junit4] 2> 2601042 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2601042 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2601042 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-15T18:32:18.764Z
[junit4] 2> 2601047 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2601047 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/control-001/solr.xml
[junit4] 2> 2601067 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2601068 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56046/solr
[junit4] 2> 2601111 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56049_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2601113 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56049_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56049_
[junit4] 2> 2601116 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56049_ ] o.a.s.c.Overseer Overseer (id=97622605058146308-127.0.0.1:56049_-n_0000000000) starting
[junit4] 2> 2601131 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56049_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56049_
[junit4] 2> 2601136 INFO (zkCallback-4908-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2601222 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56049_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/control-001/cores
[junit4] 2> 2601223 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56049_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2601227 INFO (OverseerStateUpdate-97622605058146308-127.0.0.1:56049_-n_0000000000) [n:127.0.0.1:56049_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2602252 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
[junit4] 2> 2602283 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2602385 WARN (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2602387 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2602410 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 2602410 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2602411 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/control-001/cores/collection1/data/]
[junit4] 2> 2602411 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5392bbac
[junit4] 2> 2602413 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=96.6728515625, floorSegmentMB=0.73046875, forceMergeDeletesPctAllowed=9.382267522105154, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 2602417 WARN (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2602496 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2602496 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2602497 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2602497 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2602498 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4619328371495077]
[junit4] 2> 2602498 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7d3998e6[collection1] main]
[junit4] 2> 2602500 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2602501 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2602501 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2602502 INFO (searcherExecutor-9180-thread-1-processing-n:127.0.0.1:56049_ x:collection1 c:control_collection) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7d3998e6[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2602503 INFO (coreLoadExecutor-9179-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1561961682934169600
[junit4] 2> 2602514 INFO (coreZkRegister-9172-thread-1-processing-n:127.0.0.1:56049_ x:collection1 c:control_collection) [n:127.0.0.1:56049_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2602514 INFO (coreZkRegister-9172-thread-1-processing-n:127.0.0.1:56049_ x:collection1 c:control_collection) [n:127.0.0.1:56049_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2602515 INFO (coreZkRegister-9172-thread-1-processing-n:127.0.0.1:56049_ x:collection1 c:control_collection) [n:127.0.0.1:56049_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56049/collection1/
[junit4] 2> 2602515 INFO (coreZkRegister-9172-thread-1-processing-n:127.0.0.1:56049_ x:collection1 c:control_collection) [n:127.0.0.1:56049_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2602515 INFO (coreZkRegister-9172-thread-1-processing-n:127.0.0.1:56049_ x:collection1 c:control_collection) [n:127.0.0.1:56049_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:56049/collection1/ has no replicas
[junit4] 2> 2602515 INFO (coreZkRegister-9172-thread-1-processing-n:127.0.0.1:56049_ x:collection1 c:control_collection) [n:127.0.0.1:56049_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2602523 INFO (coreZkRegister-9172-thread-1-processing-n:127.0.0.1:56049_ x:collection1 c:control_collection) [n:127.0.0.1:56049_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56049/collection1/ shard1
[junit4] 2> 2602593 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2602598 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56046/solr ready
[junit4] 2> 2602598 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 2602682 INFO (coreZkRegister-9172-thread-1-processing-n:127.0.0.1:56049_ x:collection1 c:control_collection) [n:127.0.0.1:56049_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2603360 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-1-001/cores/collection1
[junit4] 2> 2603362 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-1-001
[junit4] 2> 2603363 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2603364 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@368e8207{/,null,AVAILABLE}
[junit4] 2> 2603369 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@404a9058{HTTP/1.1,[http/1.1]}{127.0.0.1:56059}
[junit4] 2> 2603369 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.Server Started @2610153ms
[junit4] 2> 2603369 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=56059, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-1-001/cores}
[junit4] 2> 2603369 ERROR (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2603371 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.5.0
[junit4] 2> 2603371 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2603371 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2603371 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-15T18:32:21.093Z
[junit4] 2> 2603376 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2603376 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-1-001/solr.xml
[junit4] 2> 2603416 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2603417 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56046/solr
[junit4] 2> 2603435 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56059_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2603440 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56059_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2603444 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56059_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56059_
[junit4] 2> 2603448 INFO (zkCallback-4912-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2603448 INFO (zkCallback-4908-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2603449 INFO (zkCallback-4917-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2603531 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56059_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-1-001/cores
[junit4] 2> 2603531 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56059_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2603534 INFO (OverseerStateUpdate-97622605058146308-127.0.0.1:56049_-n_0000000000) [n:127.0.0.1:56049_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
[junit4] 2> 2604556 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
[junit4] 2> 2604569 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2604656 WARN (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2604658 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2604683 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2604684 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2604684 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 2604684 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5392bbac
[junit4] 2> 2604685 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=96.6728515625, floorSegmentMB=0.73046875, forceMergeDeletesPctAllowed=9.382267522105154, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 2604691 WARN (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2604752 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2604752 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2604753 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2604754 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2604755 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4619328371495077]
[junit4] 2> 2604755 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@97a4b64[collection1] main]
[junit4] 2> 2604757 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2604757 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2604757 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2604758 INFO (searcherExecutor-9191-thread-1-processing-n:127.0.0.1:56059_ x:collection1 c:collection1) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@97a4b64[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2604759 INFO (coreLoadExecutor-9190-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1561961685299757056
[junit4] 2> 2604769 INFO (coreZkRegister-9185-thread-1-processing-n:127.0.0.1:56059_ x:collection1 c:collection1) [n:127.0.0.1:56059_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2604769 INFO (coreZkRegister-9185-thread-1-processing-n:127.0.0.1:56059_ x:collection1 c:collection1) [n:127.0.0.1:56059_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2604769 INFO (coreZkRegister-9185-thread-1-processing-n:127.0.0.1:56059_ x:collection1 c:collection1) [n:127.0.0.1:56059_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56059/collection1/
[junit4] 2> 2604769 INFO (coreZkRegister-9185-thread-1-processing-n:127.0.0.1:56059_ x:collection1 c:collection1) [n:127.0.0.1:56059_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2604769 INFO (coreZkRegister-9185-thread-1-processing-n:127.0.0.1:56059_ x:collection1 c:collection1) [n:127.0.0.1:56059_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:56059/collection1/ has no replicas
[junit4] 2> 2604770 INFO (coreZkRegister-9185-thread-1-processing-n:127.0.0.1:56059_ x:collection1 c:collection1) [n:127.0.0.1:56059_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2604776 INFO (coreZkRegister-9185-thread-1-processing-n:127.0.0.1:56059_ x:collection1 c:collection1) [n:127.0.0.1:56059_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56059/collection1/ shard2
[junit4] 2> 2604933 INFO (coreZkRegister-9185-thread-1-processing-n:127.0.0.1:56059_ x:collection1 c:collection1) [n:127.0.0.1:56059_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2605159 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-2-001/cores/collection1
[junit4] 2> 2605183 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-2-001
[junit4] 2> 2605184 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2605186 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2303e09f{/,null,AVAILABLE}
[junit4] 2> 2605186 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@540a9ccd{HTTP/1.1,[http/1.1]}{127.0.0.1:56068}
[junit4] 2> 2605186 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.Server Started @2611971ms
[junit4] 2> 2605187 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=56068, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-2-001/cores}
[junit4] 2> 2605187 ERROR (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2605187 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.5.0
[junit4] 2> 2605187 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2605188 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2605188 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-15T18:32:22.910Z
[junit4] 2> 2605191 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2605191 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-2-001/solr.xml
[junit4] 2> 2605203 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2605204 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56046/solr
[junit4] 2> 2605223 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56068_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2605228 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56068_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2605232 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56068_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56068_
[junit4] 2> 2605235 INFO (zkCallback-4912-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2605235 INFO (zkCallback-4908-thread-3-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2605235 INFO (zkCallback-4917-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2605236 INFO (zkCallback-4923-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2605334 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56068_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-2-001/cores
[junit4] 2> 2605335 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56068_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2605337 INFO (OverseerStateUpdate-97622605058146308-127.0.0.1:56049_-n_0000000000) [n:127.0.0.1:56049_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2606357 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
[junit4] 2> 2606377 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2606479 WARN (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2606481 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2606499 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2606500 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2606500 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 2606500 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5392bbac
[junit4] 2> 2606503 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=96.6728515625, floorSegmentMB=0.73046875, forceMergeDeletesPctAllowed=9.382267522105154, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 2606508 WARN (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2606584 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2606584 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2606586 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2606586 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2606586 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4619328371495077]
[junit4] 2> 2606590 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@13431111[collection1] main]
[junit4] 2> 2606592 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2606593 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2606593 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2606594 INFO (searcherExecutor-9202-thread-1-processing-n:127.0.0.1:56068_ x:collection1 c:collection1) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@13431111[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2606595 INFO (coreLoadExecutor-9201-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1561961687224942592
[junit4] 2> 2606605 INFO (coreZkRegister-9196-thread-1-processing-n:127.0.0.1:56068_ x:collection1 c:collection1) [n:127.0.0.1:56068_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2606605 INFO (coreZkRegister-9196-thread-1-processing-n:127.0.0.1:56068_ x:collection1 c:collection1) [n:127.0.0.1:56068_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2606605 INFO (coreZkRegister-9196-thread-1-processing-n:127.0.0.1:56068_ x:collection1 c:collection1) [n:127.0.0.1:56068_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56068/collection1/
[junit4] 2> 2606605 INFO (coreZkRegister-9196-thread-1-processing-n:127.0.0.1:56068_ x:collection1 c:collection1) [n:127.0.0.1:56068_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2606605 INFO (coreZkRegister-9196-thread-1-processing-n:127.0.0.1:56068_ x:collection1 c:collection1) [n:127.0.0.1:56068_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:56068/collection1/ has no replicas
[junit4] 2> 2606605 INFO (coreZkRegister-9196-thread-1-processing-n:127.0.0.1:56068_ x:collection1 c:collection1) [n:127.0.0.1:56068_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2606611 INFO (coreZkRegister-9196-thread-1-processing-n:127.0.0.1:56068_ x:collection1 c:collection1) [n:127.0.0.1:56068_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56068/collection1/ shard1
[junit4] 2> 2606772 INFO (coreZkRegister-9196-thread-1-processing-n:127.0.0.1:56068_ x:collection1 c:collection1) [n:127.0.0.1:56068_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2607336 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-3-001/cores/collection1
[junit4] 2> 2607342 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-3-001
[junit4] 2> 2607353 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2607365 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1feb54b0{/,null,AVAILABLE}
[junit4] 2> 2607365 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4864c6c2{HTTP/1.1,[http/1.1]}{127.0.0.1:56084}
[junit4] 2> 2607366 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.Server Started @2614151ms
[junit4] 2> 2607366 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=56084, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-3-001/cores}
[junit4] 2> 2607366 ERROR (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2607367 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.5.0
[junit4] 2> 2607367 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2607367 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2607367 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-15T18:32:25.089Z
[junit4] 2> 2607371 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2607371 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-3-001/solr.xml
[junit4] 2> 2607393 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2607394 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56046/solr
[junit4] 2> 2607412 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56084_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2607417 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56084_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2607421 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56084_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56084_
[junit4] 2> 2607424 INFO (zkCallback-4923-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2607424 INFO (zkCallback-4908-thread-3-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2607424 INFO (zkCallback-4917-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2607424 INFO (zkCallback-4912-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2607425 INFO (zkCallback-4929-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2607565 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56084_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-3-001/cores
[junit4] 2> 2607565 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56084_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2607569 INFO (OverseerStateUpdate-97622605058146308-127.0.0.1:56049_-n_0000000000) [n:127.0.0.1:56049_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
[junit4] 2> 2608600 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
[junit4] 2> 2608615 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2608793 WARN (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2608795 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2608819 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2608819 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2608819 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 2608820 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5392bbac
[junit4] 2> 2608832 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=96.6728515625, floorSegmentMB=0.73046875, forceMergeDeletesPctAllowed=9.382267522105154, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 2608906 WARN (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2608977 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2608978 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2608979 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2608979 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2608980 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4619328371495077]
[junit4] 2> 2608980 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@143a71eb[collection1] main]
[junit4] 2> 2608982 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2608982 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2608982 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2608983 INFO (searcherExecutor-9213-thread-1-processing-n:127.0.0.1:56084_ x:collection1 c:collection1) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@143a71eb[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2608984 INFO (coreLoadExecutor-9212-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1561961689729990656
[junit4] 2> 2608990 INFO (coreZkRegister-9207-thread-1-processing-n:127.0.0.1:56084_ x:collection1 c:collection1) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 2608990 INFO (updateExecutor-4926-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2608990 INFO (recoveryExecutor-4927-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2608991 INFO (recoveryExecutor-4927-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2608991 INFO (recoveryExecutor-4927-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 2608991 INFO (recoveryExecutor-4927-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2608991 INFO (recoveryExecutor-4927-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:56059/collection1/] and I am [http://127.0.0.1:56084/collection1/]
[junit4] 2> 2608995 INFO (recoveryExecutor-4927-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:56059]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:56084_&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2608999 INFO (qtp1438256287-20904) [n:127.0.0.1:56059_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 2609000 INFO (qtp1438256287-20904) [n:127.0.0.1:56059_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard2 of collection1) have state: recovering
[junit4] 2> 2609000 INFO (qtp1438256287-20904) [n:127.0.0.1:56059_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:56084_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:56084","node_name":"127.0.0.1:56084_","state":"down"}
[junit4] 2> 2609643 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-4-001/cores/collection1
[junit4] 2> 2609644 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-4-001
[junit4] 2> 2609645 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2609647 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7e2e54ab{/,null,AVAILABLE}
[junit4] 2> 2609648 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@25b7636c{HTTP/1.1,[http/1.1]}{127.0.0.1:56093}
[junit4] 2> 2609648 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.Server Started @2616432ms
[junit4] 2> 2609648 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/tempDir-001/jetty4, solrconfig=solrconfig.xml, hostContext=/, hostPort=56093, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-4-001/cores}
[junit4] 2> 2609649 ERROR (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2609649 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.5.0
[junit4] 2> 2609649 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2609649 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2609649 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-15T18:32:27.371Z
[junit4] 2> 2609653 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2609653 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-4-001/solr.xml
[junit4] 2> 2609672 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2609673 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56046/solr
[junit4] 2> 2609691 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56093_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 2609696 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56093_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2609701 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56093_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56093_
[junit4] 2> 2609704 INFO (zkCallback-4912-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2609704 INFO (zkCallback-4923-thread-1-processing-n:127.0.0.1:56068_) [n:127.0.0.1:56068_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2609704 INFO (zkCallback-4908-thread-2-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2609704 INFO (zkCallback-4929-thread-1-processing-n:127.0.0.1:56084_) [n:127.0.0.1:56084_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2609704 INFO (zkCallback-4917-thread-1-processing-n:127.0.0.1:56059_) [n:127.0.0.1:56059_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2609705 INFO (zkCallback-4936-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2609815 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56093_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-4-001/cores
[junit4] 2> 2609815 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [n:127.0.0.1:56093_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2609822 INFO (OverseerStateUpdate-97622605058146308-127.0.0.1:56049_-n_0000000000) [n:127.0.0.1:56049_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2610006 INFO (qtp1438256287-20904) [n:127.0.0.1:56059_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:56084_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:56084","node_name":"127.0.0.1:56084_","state":"recovering"}
[junit4] 2> 2610006 INFO (qtp1438256287-20904) [n:127.0.0.1:56059_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 2610006 INFO (qtp1438256287-20904) [n:127.0.0.1:56059_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:56084_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1007
[junit4] 2> 2610513 INFO (recoveryExecutor-4927-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:56059/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 2610514 INFO (recoveryExecutor-4927-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:56084 START replicas=[http://127.0.0.1:56059/collection1/] nUpdates=100
[junit4] 2> 2610520 INFO (qtp1438256287-20905) [n:127.0.0.1:56059_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:3.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2610520 INFO (qtp1438256287-20905) [n:127.0.0.1:56059_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=4
[junit4] 2> 2610527 INFO (recoveryExecutor-4927-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:5.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2610527 INFO (recoveryExecutor-4927-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2610527 INFO (recoveryExecutor-4927-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2610527 INFO (recoveryExecutor-4927-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2610528 INFO (recoveryExecutor-4927-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2610528 INFO (recoveryExecutor-4927-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2610528 INFO (recoveryExecutor-4927-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2610528 INFO (recoveryExecutor-4927-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2610528 INFO (recoveryExecutor-4927-thread-1-processing-n:127.0.0.1:56084_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:56084_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2610840 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
[junit4] 2> 2610855 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2610971 WARN (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2610973 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2610990 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2610991 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2610991 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-4-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001/shard-4-001/cores/collection1/data/]
[junit4] 2> 2610991 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5392bbac
[junit4] 2> 2610995 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=96.6728515625, floorSegmentMB=0.73046875, forceMergeDeletesPctAllowed=9.382267522105154, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 2611002 WARN (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2611074 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2611074 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2611075 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2611075 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2611076 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4619328371495077]
[junit4] 2> 2611076 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@73f88921[collection1] main]
[junit4] 2> 2611078 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2611079 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2611079 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2611080 INFO (searcherExecutor-9224-thread-1-processing-n:127.0.0.1:56093_ x:collection1 c:collection1) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@73f88921[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2611082 INFO (coreLoadExecutor-9223-thread-1-processing-n:127.0.0.1:56093_) [n:127.0.0.1:56093_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1561961691929903104
[junit4] 2> 2611087 INFO (coreZkRegister-9218-thread-1-processing-n:127.0.0.1:56093_ x:collection1 c:collection1) [n:127.0.0.1:56093_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 2611088 INFO (updateExecutor-4933-thread-1-processing-n:127.0.0.1:56093_ x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:56093_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2611088 INFO (recoveryExecutor-4934-thread-1-processing-n:127.0.0.1:56093_ x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:56093_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2611089 INFO (recoveryExecutor-4934-thread-1-processing-n:127.0.0.1:56093_ x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:56093_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2611089 INFO (recoveryExecutor-4934-thread-1-processing-n:127.0.0.1:56093_ x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:56093_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 2611089 INFO (recoveryExecutor-4934-thread-1-processing-n:127.0.0.1:56093_ x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:56093_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2611090 INFO (recoveryExecutor-4934-thread-1-processing-n:127.0.0.1:56093_ x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:56093_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:56068/collection1/] and I am [http://127.0.0.1:56093/collection1/]
[junit4] 2> 2611094 INFO (recoveryExecutor-4934-thread-1-processing-n:127.0.0.1:56093_ x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:56093_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:56068]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:56093_&coreNodeName=core_node4&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2611096 INFO (qtp1634710610-20932) [n:127.0.0.1:56068_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node4, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 2611097 INFO (qtp1634710610-20932) [n:127.0.0.1:56068_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 2611097 INFO (qtp1634710610-20932) [n:127.0.0.1:56068_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:56093_, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{"core":"collection1","base_url":"http://127.0.0.1:56093","node_name":"127.0.0.1:56093_","state":"down"}
[junit4] 2> 2611173 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2611175 INFO (qtp1438256287-20908) [n:127.0.0.1:56059_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params async=1&collection.configName=conf1&name=ocptest1&action=CREATE&numShards=4&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2611178 INFO (qtp1438256287-20908) [n:127.0.0.1:56059_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={async=1&collection.configName=conf1&name=ocptest1&action=CREATE&numShards=4&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 2611180 INFO (qtp1438256287-20909) [n:127.0.0.1:56059_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params async=2&collection.configName=conf1&name=ocptest2&action=CREATE&numShards=4&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2611182 INFO (OverseerThreadFactory-9177-thread-1-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ ] o.a.s.c.CreateCollectionCmd Create collection ocptest1
[junit4] 2> 2611183 INFO (qtp1438256287-20909) [n:127.0.0.1:56059_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={async=2&collection.configName=conf1&name=ocptest2&action=CREATE&numShards=4&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 2611185 INFO (qtp1438256287-20902) [n:127.0.0.1:56059_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params async=3&collection.configName=conf1&name=ocptest3&action=CREATE&numShards=4&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2611189 INFO (OverseerThreadFactory-9177-thread-2-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ ] o.a.s.c.CreateCollectionCmd Create collection ocptest2
[junit4] 2> 2611190 INFO (qtp1438256287-20902) [n:127.0.0.1:56059_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={async=3&collection.configName=conf1&name=ocptest3&action=CREATE&numShards=4&wt=javabin&version=2} status=0 QTime=4
[junit4] 2> 2611192 INFO (qtp1438256287-20904) [n:127.0.0.1:56059_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params async=4&collection.configName=conf1&name=ocptest4&action=CREATE&numShards=4&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2611199 INFO (OverseerThreadFactory-9177-thread-3-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ ] o.a.s.c.CreateCollectionCmd Create collection ocptest3
[junit4] 2> 2611200 INFO (qtp1438256287-20904) [n:127.0.0.1:56059_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={async=4&collection.configName=conf1&name=ocptest4&action=CREATE&numShards=4&wt=javabin&version=2} status=0 QTime=8
[junit4] 2> 2611202 INFO (qtp1438256287-20906) [n:127.0.0.1:56059_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=1&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2611205 INFO (qtp1438256287-20906) [n:127.0.0.1:56059_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=1&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 2611207 INFO (OverseerThreadFactory-9177-thread-4-processing-n:127.0.0.1:56049_) [n:127.0.0.1:56049_ ] o.a.s.c.CreateCollectionCmd Create collection ocptest4
[junit4] 2> 2611207 INFO (qtp1438256287-20905) [n:127.0.0.1:56059_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=2&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCP
[...truncated too long message...]
Closing metric reporters for: solr.core.collection1
[junit4] 2> 2643932 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 2643932 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@25b7636c{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 2643933 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7e2e54ab{/,null,UNAVAILABLE}
[junit4] 2> 2643935 INFO (TEST-MultiThreadedOCPTest.test-seed#[AA2BD94A175182ED]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:56046 56046
[junit4] 2> 2643979 INFO (Thread-3509) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:56046 56046
[junit4] 2> 2650272 WARN (Thread-3509) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 5 /solr/security.json
[junit4] 2> 5 /solr/configs/conf1
[junit4] 2> 4 /solr/collections/collection1/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 4 /solr/collections/ocptest_shardsplit2/state.json
[junit4] 2> 4 /solr/collections/ocptest1/state.json
[junit4] 2> 4 /solr/collections/ocptest3/state.json
[junit4] 2> 4 /solr/collections/ocptest2/state.json
[junit4] 2> 4 /solr/collections/ocptest4/state.json
[junit4] 2> 4 /solr/collections/ocptest_shardsplit/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 58 /solr/overseer/queue
[junit4] 2> 14 /solr/overseer/collection-queue-work
[junit4] 2> 11 /solr/overseer/queue-work
[junit4] 2> 6 /solr/live_nodes
[junit4] 2> 6 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=MultiThreadedOCPTest -Dtests.method=test -Dtests.seed=AA2BD94A175182ED -Dtests.slow=true -Dtests.locale=lt-LT -Dtests.timezone=America/St_Thomas -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 49.7s J1 | MultiThreadedOCPTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: Task 3001 did not complete, final state: FAILED expected same:<COMPLETED> was not:<FAILED>
[junit4] > at __randomizedtesting.SeedInfo.seed([AA2BD94A175182ED:227FE690B9ADEF15]:0)
[junit4] > at org.apache.solr.cloud.MultiThreadedOCPTest.testDeduplicationOfSubmittedTasks(MultiThreadedOCPTest.java:250)
[junit4] > at org.apache.solr.cloud.MultiThreadedOCPTest.test(MultiThreadedOCPTest.java:64)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2650313 INFO (SUITE-MultiThreadedOCPTest-seed#[AA2BD94A175182ED]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_AA2BD94A175182ED-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=944, maxMBSortInHeap=5.031440992386621, sim=RandomSimilarity(queryNorm=false,coord=no): {}, locale=lt-LT, timezone=America/St_Thomas
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=146397072,total=535298048
[junit4] 2> NOTE: All tests run in this JVM: [CdcrReplicationHandlerTest, EnumFieldTest, TestRequestStatusCollectionAPI, BigEndianAscendingWordDeserializerTest, TestBlendedInfixSuggestions, TestFreeTextSuggestions, TestManagedSynonymFilterFactory, TestSolrCoreSnapshots, TestOmitPositions, TestBinaryResponseWriter, TestRTimerTree, WordBreakSolrSpellCheckerTest, DistributedQueryComponentCustomSortTest, StatsComponentTest, DeleteInactiveReplicaTest, SimpleMLTQParserTest, TestNumericTerms32, TriLevelCompositeIdRoutingTest, TestConfigsApi, TestRandomFaceting, DeleteShardTest, ReturnFieldsTest, TestPivotHelperCode, TestMergePolicyConfig, CircularListTest, ClassificationUpdateProcessorTest, QueryParsingTest, DeleteNodeTest, TestCrossCoreJoin, AsyncCallRequestStatusResponseTest, TestSegmentSorting, TestCollapseQParserPlugin, TestRealTimeGet, ConfigSetsAPITest, TestQuerySenderListener, TestNamedUpdateProcessors, TestManagedSchemaAPI, CdcrVersionReplicationTest, TestUseDocValuesAsStored2, TestQueryTypes, OutOfBoxZkACLAndCredentialsProvidersTest, TestIndexSearcher, TestSolrConfigHandlerCloud, EchoParamsTest, DistributedFacetExistsSmallTest, TestLMDirichletSimilarityFactory, JsonValidatorTest, DirectoryFactoryTest, MissingSegmentRecoveryTest, DefaultValueUpdateProcessorTest, TestSchemaVersionResource, BasicZkTest, TestRestManager, TestCryptoKeys, TestSystemCollAutoCreate, JsonLoaderTest, FieldAnalysisRequestHandlerTest, ExternalFileFieldSortTest, TestSolr4Spatial, BlockJoinFacetSimpleTest, SpellCheckComponentTest, TestManagedResource, TestAtomicUpdateErrorCases, MinimalSchemaTest, TestReqParamsAPI, TestInPlaceUpdatesDistrib, ShowFileRequestHandlerTest, TestImpersonationWithHadoopAuth, DistributedIntervalFacetingTest, WrapperMergePolicyFactoryTest, TestExclusionRuleCollectionAccess, BitVectorTest, AnalyticsQueryTest, TestUtils, SmileWriterTest, DateMathParserTest, TestRawTransformer, TestCustomStream, TestQueryUtils, LeaderElectionIntegrationTest, MergeStrategyTest, TestSystemIdResolver, LeaderFailureAfterFreshStartTest, RegexBoostProcessorTest, TestInitParams, TestSchemaResource, ShardRoutingCustomTest, HLLUtilTest, TestMacroExpander, DistributedTermsComponentTest, BadIndexSchemaTest, SolrJmxReporterTest, TestFieldSortValues, DistanceFunctionTest, HdfsDirectoryFactoryTest, TestZkChroot, ClassificationUpdateProcessorFactoryTest, TestReplicaProperties, TestRandomCollapseQParserPlugin, SortByFunctionTest, CacheHeaderTest, SolrXmlInZkTest, TestRangeQuery, TestSolrQueryParser, LoggingHandlerTest, LeaderInitiatedRecoveryOnShardRestartTest, TestSolrDeletionPolicy2, AddSchemaFieldsUpdateProcessorFactoryTest, TestReRankQParserPlugin, CdcrBootstrapTest, RestartWhileUpdatingTest, TestExactStatsCache, TestValueSourceCache, SuggestComponentContextFilterQueryTest, DistribJoinFromCollectionTest, HdfsUnloadDistributedZkTest, ClusterStateTest, TestTestInjection, CurrencyFieldOpenExchangeTest, TestLockTree, DirectUpdateHandlerOptimizeTest, AnalysisErrorHandlingTest, TestBlobHandler, TestFiltering, TestDocSet, TestHdfsBackupRestoreCore, SegmentsInfoRequestHandlerTest, TestJsonRequest, TestRandomRequestDistribution, BlockJoinFacetDistribTest, BasicAuthIntegrationTest, TestBadConfig, SliceStateTest, ShufflingReplicaListTransformerTest, HdfsThreadLeakTest, TestLeaderInitiatedRecoveryThread, TestPKIAuthenticationPlugin, PreAnalyzedFieldManagedSchemaCloudTest, TestDistributedGrouping, TestMissingGroups, CSVRequestHandlerTest, TestSuggestSpellingConverter, SolrGangliaReporterTest, TestPointFields, TestSubQueryTransformerCrossCore, TestConfig, TestDynamicFieldResource, IndexSchemaTest, TestGraphMLResponseWriter, OverseerRolesTest, SolrCLIZkUtilsTest, TestOnReconnectListenerSupport, TestMinMaxOnMultiValuedField, DistributedFacetPivotSmallTest, ConnectionReuseTest, TestXmlQParser, TestLMJelinekMercerSimilarityFactory, DateRangeFieldTest, PreAnalyzedFieldTest, TestCollationFieldDocValues, TestDFISimilarityFactory, SolrInfoMBeanTest, TestCoreDiscovery, SolrIndexMetricsTest, TestConfigSetImmutable, UpdateLogTest, TestPushWriter, TestFieldCacheSanityChecker, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, OpenCloseCoreStressTest, OverseerTest, FullSolrCloudDistribCmdsTest, ZkSolrClientTest, AutoCommitTest, TestRecovery, ZkControllerTest, TestJoin, SolrCmdDistributorTest, ConvertedLegacyTest, TestSort, BasicFunctionalityTest, TestLazyCores, CurrencyFieldXmlFileTest, SolrIndexSplitterTest, SimplePostToolTest, SignatureUpdateProcessorFactoryTest, TestExtendedDismaxParser, CoreAdminHandlerTest, TestFoldingMultitermQuery, SuggesterTSTTest, TestTrie, TestCSVLoader, SchemaVersionSpecificBehaviorTest, TestWordDelimiterFilterFactory, StatelessScriptUpdateProcessorFactoryTest, DebugComponentTest, LukeRequestHandlerTest, TestReversedWildcardFilterFactory, FileBasedSpellCheckerTest, RequestHandlersTest, TermVectorComponentTest, MoreLikeThisHandlerTest, SolrPluginUtilsTest, TestCollationField, MBeansHandlerTest, BinaryUpdateRequestHandlerTest, PingRequestHandlerTest, TestLFUCache, DirectSolrConnectionTest, TestLuceneMatchVersion, SpellPossibilityIteratorTest, TestXIncludeConfig, TestSweetSpotSimilarityFactory, TestDFRSimilarityFactory, TestIBSimilarityFactory, TestJmxMonitoredMap, ScriptEngineTest, URLClassifyProcessorTest, SpellingQueryConverterTest, RAMDirectoryFactoryTest, TestSolrJ, TestLRUCache, TestDocumentBuilder, UUIDFieldTest, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestSimpleTrackingShardHandler, TestTolerantSearch, TestPathTrie, TestEmbeddedSolrServerSchemaAPI, ActionThrottleTest, AliasIntegrationTest, CdcrReplicationDistributedZkTest, CleanupOldIndexTest, CollectionTooManyReplicasTest, CollectionsAPISolrJTest, DeleteStatusTest, DistribDocExpirationUpdateProcessorTest, DistributedVersionInfoTest, LeaderElectionContextKeyTest, MigrateRouteKeyTest, MultiThreadedOCPTest]
[junit4] Completed [504/698 (1!)] on J1 in 49.75s, 1 test, 1 failure <<< FAILURES!
[...truncated 63370 lines...]
[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 761 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/761/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
1 tests failed.
FAILED: org.apache.solr.cloud.ShardSplitTest.testSplitWithChaosMonkey
Error Message:
There are still nodes recoverying - waited for 330 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
at __randomizedtesting.SeedInfo.seed([AB7038EACB0E46E1:2057EB3B8A08ED65]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:187)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:144)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:865)
at org.apache.solr.cloud.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:437)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12686 lines...]
[junit4] Suite: org.apache.solr.cloud.ShardSplitTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/init-core-data-001
[junit4] 2> 2735020 WARN (SUITE-ShardSplitTest-seed#[AB7038EACB0E46E1]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=8 numCloses=8
[junit4] 2> 2735020 INFO (SUITE-ShardSplitTest-seed#[AB7038EACB0E46E1]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields
[junit4] 2> 2735031 INFO (SUITE-ShardSplitTest-seed#[AB7038EACB0E46E1]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 2735033 INFO (SUITE-ShardSplitTest-seed#[AB7038EACB0E46E1]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 2735037 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2735037 INFO (Thread-3376) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2735037 INFO (Thread-3376) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2735144 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.ZkTestServer start zk server on port:52289
[junit4] 2> 2735186 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 2735207 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 2735229 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 2735233 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 2735244 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 2735252 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 2735257 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 2735263 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 2735267 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 2735271 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 2735275 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 2737277 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/control-001/cores/collection1
[junit4] 2> 2737280 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2737282 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@a9fd763{/,null,AVAILABLE}
[junit4] 2> 2737282 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6f1e35a9{HTTP/1.1,[http/1.1]}{127.0.0.1:52292}
[junit4] 2> 2737283 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.Server Started @2743151ms
[junit4] 2> 2737283 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/tempDir-001/control/data, hostContext=/, hostPort=52292, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/control-001/cores}
[junit4] 2> 2737283 ERROR (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2737283 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.5.0
[junit4] 2> 2737283 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2737283 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2737284 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-15T07:07:27.352Z
[junit4] 2> 2737290 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2737290 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/control-001/solr.xml
[junit4] 2> 2737310 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2737311 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52289/solr
[junit4] 2> 2737365 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52292_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2737366 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52292_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:52292_
[junit4] 2> 2737376 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52292_ ] o.a.s.c.Overseer Overseer (id=97619911973928964-127.0.0.1:52292_-n_0000000000) starting
[junit4] 2> 2737391 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52292_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52292_
[junit4] 2> 2737398 INFO (zkCallback-2704-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2737568 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52292_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/control-001/cores
[junit4] 2> 2737568 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52292_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2737572 INFO (OverseerStateUpdate-97619911973928964-127.0.0.1:52292_-n_0000000000) [n:127.0.0.1:52292_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2738602 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
[junit4] 2> 2738613 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2738685 WARN (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2738687 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 2738718 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 2738718 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2738718 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/control-001/cores/collection1/data/]
[junit4] 2> 2738719 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7308330b
[junit4] 2> 2738725 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=758933313, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2738750 WARN (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2738865 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2738865 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2738866 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2738866 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2738867 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=1.2607421875, floorSegmentMB=0.4091796875, forceMergeDeletesPctAllowed=23.162442070192352, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4588509067382843
[junit4] 2> 2738868 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@76b4ee80[collection1] main]
[junit4] 2> 2738869 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2738870 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2738870 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2738871 INFO (coreLoadExecutor-8340-thread-1-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1561918595597860864
[junit4] 2> 2738874 INFO (searcherExecutor-8341-thread-1-processing-n:127.0.0.1:52292_ x:collection1 c:control_collection) [n:127.0.0.1:52292_ c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@76b4ee80[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2738883 INFO (coreZkRegister-8333-thread-1-processing-n:127.0.0.1:52292_ x:collection1 c:control_collection) [n:127.0.0.1:52292_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2738884 INFO (coreZkRegister-8333-thread-1-processing-n:127.0.0.1:52292_ x:collection1 c:control_collection) [n:127.0.0.1:52292_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2738884 INFO (coreZkRegister-8333-thread-1-processing-n:127.0.0.1:52292_ x:collection1 c:control_collection) [n:127.0.0.1:52292_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52292/collection1/
[junit4] 2> 2738884 INFO (coreZkRegister-8333-thread-1-processing-n:127.0.0.1:52292_ x:collection1 c:control_collection) [n:127.0.0.1:52292_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2738884 INFO (coreZkRegister-8333-thread-1-processing-n:127.0.0.1:52292_ x:collection1 c:control_collection) [n:127.0.0.1:52292_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:52292/collection1/ has no replicas
[junit4] 2> 2738884 INFO (coreZkRegister-8333-thread-1-processing-n:127.0.0.1:52292_ x:collection1 c:control_collection) [n:127.0.0.1:52292_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2738893 INFO (coreZkRegister-8333-thread-1-processing-n:127.0.0.1:52292_ x:collection1 c:control_collection) [n:127.0.0.1:52292_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52292/collection1/ shard1
[junit4] 2> 2739060 INFO (coreZkRegister-8333-thread-1-processing-n:127.0.0.1:52292_ x:collection1 c:control_collection) [n:127.0.0.1:52292_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2739061 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2739063 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52289/solr ready
[junit4] 2> 2739063 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 2739064 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
[junit4] 2> 2739418 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-1-001/cores/collection1
[junit4] 2> 2739419 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-1-001
[junit4] 2> 2739420 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2739422 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@690cf968{/,null,AVAILABLE}
[junit4] 2> 2739422 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@224152b6{HTTP/1.1,[http/1.1]}{127.0.0.1:52298}
[junit4] 2> 2739422 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.Server Started @2745291ms
[junit4] 2> 2739422 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=52298, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-1-001/cores}
[junit4] 2> 2739423 ERROR (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2739424 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.5.0
[junit4] 2> 2739424 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2739424 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2739424 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-15T07:07:29.492Z
[junit4] 2> 2739428 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2739428 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-1-001/solr.xml
[junit4] 2> 2739486 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2739487 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52289/solr
[junit4] 2> 2739505 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2739512 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52298_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2739516 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52298_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52298_
[junit4] 2> 2739521 INFO (zkCallback-2708-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2739521 INFO (zkCallback-2704-thread-3-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2739523 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2739630 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52298_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-1-001/cores
[junit4] 2> 2739630 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52298_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2739637 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2739641 INFO (OverseerStateUpdate-97619911973928964-127.0.0.1:52292_-n_0000000000) [n:127.0.0.1:52292_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
[junit4] 2> 2739759 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2740663 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
[junit4] 2> 2740683 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2740747 WARN (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2740749 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 2740781 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2740781 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2740781 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 2740782 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7308330b
[junit4] 2> 2740785 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=758933313, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2740798 WARN (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2740907 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2740907 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2740908 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2740908 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2740910 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=1.2607421875, floorSegmentMB=0.4091796875, forceMergeDeletesPctAllowed=23.162442070192352, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4588509067382843
[junit4] 2> 2740916 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@cbbdbbf[collection1] main]
[junit4] 2> 2740918 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2740919 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2740919 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2740920 INFO (searcherExecutor-8352-thread-1-processing-n:127.0.0.1:52298_ x:collection1 c:collection1) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@cbbdbbf[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2740921 INFO (coreLoadExecutor-8351-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1561918597747441664
[junit4] 2> 2740932 INFO (coreZkRegister-8346-thread-1-processing-n:127.0.0.1:52298_ x:collection1 c:collection1) [n:127.0.0.1:52298_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2740932 INFO (coreZkRegister-8346-thread-1-processing-n:127.0.0.1:52298_ x:collection1 c:collection1) [n:127.0.0.1:52298_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2740932 INFO (coreZkRegister-8346-thread-1-processing-n:127.0.0.1:52298_ x:collection1 c:collection1) [n:127.0.0.1:52298_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52298/collection1/
[junit4] 2> 2740932 INFO (coreZkRegister-8346-thread-1-processing-n:127.0.0.1:52298_ x:collection1 c:collection1) [n:127.0.0.1:52298_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2740932 INFO (coreZkRegister-8346-thread-1-processing-n:127.0.0.1:52298_ x:collection1 c:collection1) [n:127.0.0.1:52298_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:52298/collection1/ has no replicas
[junit4] 2> 2740932 INFO (coreZkRegister-8346-thread-1-processing-n:127.0.0.1:52298_ x:collection1 c:collection1) [n:127.0.0.1:52298_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2740943 INFO (coreZkRegister-8346-thread-1-processing-n:127.0.0.1:52298_ x:collection1 c:collection1) [n:127.0.0.1:52298_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52298/collection1/ shard2
[junit4] 2> 2741052 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2741104 INFO (coreZkRegister-8346-thread-1-processing-n:127.0.0.1:52298_ x:collection1 c:collection1) [n:127.0.0.1:52298_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2741213 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2741475 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-2-001/cores/collection1
[junit4] 2> 2741476 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-2-001
[junit4] 2> 2741477 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2741479 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@49106bb8{/,null,AVAILABLE}
[junit4] 2> 2741479 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@661ed17b{HTTP/1.1,[http/1.1]}{127.0.0.1:52302}
[junit4] 2> 2741479 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.Server Started @2747348ms
[junit4] 2> 2741480 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=52302, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-2-001/cores}
[junit4] 2> 2741481 ERROR (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2741481 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.5.0
[junit4] 2> 2741481 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2741481 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2741481 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-15T07:07:31.549Z
[junit4] 2> 2741485 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2741485 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-2-001/solr.xml
[junit4] 2> 2741513 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2741514 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52289/solr
[junit4] 2> 2741533 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52302_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2741540 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52302_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2741544 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52302_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52302_
[junit4] 2> 2741546 INFO (zkCallback-2704-thread-3-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2741546 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2741546 INFO (zkCallback-2708-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2741547 INFO (zkCallback-2720-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2741658 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2741659 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52302_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-2-001/cores
[junit4] 2> 2741659 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52302_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2741664 INFO (OverseerStateUpdate-97619911973928964-127.0.0.1:52292_-n_0000000000) [n:127.0.0.1:52292_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2741777 INFO (zkCallback-2720-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2741777 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2742687 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
[junit4] 2> 2742716 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2742784 WARN (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2742785 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 2742814 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2742815 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2742815 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 2742815 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7308330b
[junit4] 2> 2742818 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=758933313, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2742828 WARN (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2742913 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2742913 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2742914 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2742914 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2742916 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=1.2607421875, floorSegmentMB=0.4091796875, forceMergeDeletesPctAllowed=23.162442070192352, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4588509067382843
[junit4] 2> 2742917 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@709f3d2d[collection1] main]
[junit4] 2> 2742919 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2742919 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2742919 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2742920 INFO (searcherExecutor-8363-thread-1-processing-n:127.0.0.1:52302_ x:collection1 c:collection1) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@709f3d2d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2742921 INFO (coreLoadExecutor-8362-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1561918599844593664
[junit4] 2> 2742932 INFO (coreZkRegister-8357-thread-1-processing-n:127.0.0.1:52302_ x:collection1 c:collection1) [n:127.0.0.1:52302_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2742932 INFO (coreZkRegister-8357-thread-1-processing-n:127.0.0.1:52302_ x:collection1 c:collection1) [n:127.0.0.1:52302_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2742932 INFO (coreZkRegister-8357-thread-1-processing-n:127.0.0.1:52302_ x:collection1 c:collection1) [n:127.0.0.1:52302_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52302/collection1/
[junit4] 2> 2742932 INFO (coreZkRegister-8357-thread-1-processing-n:127.0.0.1:52302_ x:collection1 c:collection1) [n:127.0.0.1:52302_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2742932 INFO (coreZkRegister-8357-thread-1-processing-n:127.0.0.1:52302_ x:collection1 c:collection1) [n:127.0.0.1:52302_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:52302/collection1/ has no replicas
[junit4] 2> 2742932 INFO (coreZkRegister-8357-thread-1-processing-n:127.0.0.1:52302_ x:collection1 c:collection1) [n:127.0.0.1:52302_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2742938 INFO (coreZkRegister-8357-thread-1-processing-n:127.0.0.1:52302_ x:collection1 c:collection1) [n:127.0.0.1:52302_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52302/collection1/ shard1
[junit4] 2> 2743054 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2743054 INFO (zkCallback-2720-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2743110 INFO (coreZkRegister-8357-thread-1-processing-n:127.0.0.1:52302_ x:collection1 c:collection1) [n:127.0.0.1:52302_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2743229 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2743229 INFO (zkCallback-2720-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2743606 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-3-001/cores/collection1
[junit4] 2> 2743608 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-3-001
[junit4] 2> 2743608 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2743611 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7f5a6e7b{/,null,AVAILABLE}
[junit4] 2> 2743611 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3b1402fa{HTTP/1.1,[http/1.1]}{127.0.0.1:52306}
[junit4] 2> 2743611 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.Server Started @2749480ms
[junit4] 2> 2743611 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=52306, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-3-001/cores}
[junit4] 2> 2743613 ERROR (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2743613 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.5.0
[junit4] 2> 2743613 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2743613 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2743613 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-15T07:07:33.681Z
[junit4] 2> 2743617 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2743617 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-3-001/solr.xml
[junit4] 2> 2743627 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2743629 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52289/solr
[junit4] 2> 2743646 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52306_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2743652 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52306_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2743656 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52306_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52306_
[junit4] 2> 2743659 INFO (zkCallback-2704-thread-3-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2743659 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2743659 INFO (zkCallback-2720-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2743660 INFO (zkCallback-2726-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2743660 INFO (zkCallback-2708-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2743769 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52306_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-3-001/cores
[junit4] 2> 2743769 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52306_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2743771 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2743771 INFO (zkCallback-2720-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2743775 INFO (OverseerStateUpdate-97619911973928964-127.0.0.1:52292_-n_0000000000) [n:127.0.0.1:52292_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
[junit4] 2> 2743884 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2743884 INFO (zkCallback-2720-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2743884 INFO (zkCallback-2726-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2744806 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
[junit4] 2> 2744834 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2744918 WARN (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2744919 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 2744952 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2744953 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2744953 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 2744953 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7308330b
[junit4] 2> 2744956 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=758933313, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2744966 WARN (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2745063 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2745063 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2745064 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2745064 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2745065 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=1.2607421875, floorSegmentMB=0.4091796875, forceMergeDeletesPctAllowed=23.162442070192352, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4588509067382843
[junit4] 2> 2745086 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4fbba45e[collection1] main]
[junit4] 2> 2745088 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2745089 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2745089 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2745090 INFO (searcherExecutor-8374-thread-1-processing-n:127.0.0.1:52306_ x:collection1 c:collection1) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4fbba45e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2745091 INFO (coreLoadExecutor-8373-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1561918602120003584
[junit4] 2> 2745097 INFO (coreZkRegister-8368-thread-1-processing-n:127.0.0.1:52306_ x:collection1 c:collection1) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 2745097 INFO (updateExecutor-2723-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2745098 INFO (recoveryExecutor-2724-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2745098 INFO (recoveryExecutor-2724-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2745098 INFO (recoveryExecutor-2724-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 2745098 INFO (recoveryExecutor-2724-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2745098 INFO (recoveryExecutor-2724-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:52298/collection1/] and I am [http://127.0.0.1:52306/collection1/]
[junit4] 2> 2745104 INFO (recoveryExecutor-2724-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:52298]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:52306_&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2745108 INFO (qtp2128269251-15921) [n:127.0.0.1:52298_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 2745109 INFO (qtp2128269251-15921) [n:127.0.0.1:52298_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard2 of collection1) have state: recovering
[junit4] 2> 2745109 INFO (qtp2128269251-15921) [n:127.0.0.1:52298_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:52306_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:52306","node_name":"127.0.0.1:52306_","state":"down"}
[junit4] 2> 2745210 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2745210 INFO (zkCallback-2726-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2745210 INFO (zkCallback-2720-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2745745 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-4-001/cores/collection1
[junit4] 2> 2745747 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-4-001
[junit4] 2> 2745749 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2745750 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@388e1e68{/,null,AVAILABLE}
[junit4] 2> 2745751 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7820b85b{HTTP/1.1,[http/1.1]}{127.0.0.1:52311}
[junit4] 2> 2745751 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.Server Started @2751619ms
[junit4] 2> 2745751 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/tempDir-001/jetty4, solrconfig=solrconfig.xml, hostContext=/, hostPort=52311, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-4-001/cores}
[junit4] 2> 2745752 ERROR (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2745752 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.5.0
[junit4] 2> 2745752 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2745752 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2745752 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-15T07:07:35.820Z
[junit4] 2> 2745756 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2745756 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-4-001/solr.xml
[junit4] 2> 2745792 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2745793 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52289/solr
[junit4] 2> 2745815 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52311_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 2745820 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52311_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2745825 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52311_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52311_
[junit4] 2> 2745828 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2745829 INFO (zkCallback-2704-thread-2-processing-n:127.0.0.1:52292_) [n:127.0.0.1:52292_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2745829 INFO (zkCallback-2720-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2745829 INFO (zkCallback-2726-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2745829 INFO (zkCallback-2708-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2745830 INFO (zkCallback-2733-thread-1-processing-n:127.0.0.1:52311_) [n:127.0.0.1:52311_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2745944 INFO (zkCallback-2726-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 2745944 INFO (zkCallback-2720-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 2745944 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 2745973 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52311_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-4-001/cores
[junit4] 2> 2745973 INFO (TEST-ShardSplitTest.test-seed#[AB7038EACB0E46E1]) [n:127.0.0.1:52311_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2745979 INFO (OverseerStateUpdate-97619911973928964-127.0.0.1:52292_-n_0000000000) [n:127.0.0.1:52292_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2746094 INFO (zkCallback-2733-thread-1-processing-n:127.0.0.1:52311_) [n:127.0.0.1:52311_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 2746094 INFO (zkCallback-2726-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 2746095 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 2746095 INFO (zkCallback-2720-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 2746135 INFO (qtp2128269251-15921) [n:127.0.0.1:52298_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:52306_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:52306","node_name":"127.0.0.1:52306_","state":"recovering"}
[junit4] 2> 2746135 INFO (qtp2128269251-15921) [n:127.0.0.1:52298_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 2746135 INFO (qtp2128269251-15921) [n:127.0.0.1:52298_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:52306_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1027
[junit4] 2> 2746662 INFO (recoveryExecutor-2724-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:52298/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 2746662 INFO (recoveryExecutor-2724-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:52306 START replicas=[http://127.0.0.1:52298/collection1/] nUpdates=100
[junit4] 2> 2746666 INFO (qtp2128269251-15923) [n:127.0.0.1:52298_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2746667 INFO (qtp2128269251-15923) [n:127.0.0.1:52298_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2746668 INFO (recoveryExecutor-2724-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2746668 INFO (recoveryExecutor-2724-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2746668 INFO (recoveryExecutor-2724-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2746668 INFO (recoveryExecutor-2724-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2746668 INFO (recoveryExecutor-2724-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2746668 INFO (recoveryExecutor-2724-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2746668 INFO (recoveryExecutor-2724-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2746668 INFO (recoveryExecutor-2724-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2746668 INFO (recoveryExecutor-2724-thread-1-processing-n:127.0.0.1:52306_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52306_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2746786 INFO (zkCallback-2733-thread-1-processing-n:127.0.0.1:52311_) [n:127.0.0.1:52311_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 2746786 INFO (zkCallback-2726-thread-1-processing-n:127.0.0.1:52306_) [n:127.0.0.1:52306_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 2746786 INFO (zkCallback-2714-thread-1-processing-n:127.0.0.1:52298_) [n:127.0.0.1:52298_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 2746786 INFO (zkCallback-2720-thread-1-processing-n:127.0.0.1:52302_) [n:127.0.0.1:52302_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 2747006 INFO (coreLoadExecutor-8384-thread-1-processing-n:127.0.0.1:52311_) [n:127.0.0.1:52311_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
[junit4] 2> 2747023 INFO (coreLoadExecutor-8384-thread-1-processing-n:127.0.0.1:52311_) [n:127.0.0.1:52311_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2747093 WARN (coreLoadExecutor-8384-thread-1-processing-n:127.0.0.1:52311_) [n:127.0.0.1:52311_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2747095 INFO (coreLoadExecutor-8384-thread-1-processing-n:127.0.0.1:52311_) [n:127.0.0.1:52311_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 2747131 INFO (coreLoadExecutor-8384-thread-1-processing-n:127.0.0.1:52311_) [n:127.0.0.1:52311_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2747131 INFO (coreLoadExecutor-8384-thread-1-processing-n:127.0.0.1:52311_) [n:127.0.0.1:52311_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2747131 INFO (coreLoadExecutor-8384-thread-1-processing-n:127.0.0.1:52311_) [n:127.0.0.1:52311_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-4-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001/shard-4-001/cores/collection1/data/]
[junit4] 2> 2747131 INFO (coreLoadExecutor-8384-thread-1-processing-n:127.0.0.1:52311_) [n:127.0.0.1:52311_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7308330b
[junit4] 2> 2747135 INFO (coreLoadExecutor-8384-thread-1-processing-n:127.0.0.1:52311_) [n:127.0.0.1:52311_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=758933313, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2747153 WARN (coreLoadExecutor-8384-thread-1-processing-n:127.0.0.1:52311_) [n:127.0.0.1:52311_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2747232 INFO (coreLoadExecutor-8384-thread-1-processing-n:127.0.0.1:52311_) [n:127.0.0.1:52311_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2747232 INFO (coreLoadExecutor-8384-thread-1-processing-n:127.0.0.1:52311_) [n:127.0.0.1:52311_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2747233 INFO (coreLoadExecutor-8384-thread-1-proc
[...truncated too long message...]
plitTest.testSplitWithChaosMonkey-seed#[AB7038EACB0E46E1]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@64cccfbc{/,null,UNAVAILABLE}
[junit4] 2> 3433402 INFO (TEST-ShardSplitTest.testSplitWithChaosMonkey-seed#[AB7038EACB0E46E1]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52958 52958
[junit4] 2> 3433412 INFO (Thread-3630) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52958 52958
[junit4] 2> 3439344 WARN (Thread-3630) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/aliases.json
[junit4] 2> 7 /solr/clusterprops.json
[junit4] 2> 6 /solr/security.json
[junit4] 2> 6 /solr/collections/collection1/state.json
[junit4] 2> 6 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/clusterstate.json
[junit4] 2> 2 /solr/overseer_elect/election/97619926225321992-127.0.0.1:52973_-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 146 /solr/overseer/collection-queue-work
[junit4] 2> 76 /solr/overseer/queue
[junit4] 2> 7 /solr/collections
[junit4] 2> 6 /solr/live_nodes
[junit4] 2> 6 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ShardSplitTest -Dtests.method=testSplitWithChaosMonkey -Dtests.seed=AB7038EACB0E46E1 -Dtests.slow=true -Dtests.locale=it -Dtests.timezone=Africa/Ouagadougou -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] FAILURE 487s J0 | ShardSplitTest.testSplitWithChaosMonkey <<<
[junit4] > Throwable #1: java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
[junit4] > at __randomizedtesting.SeedInfo.seed([AB7038EACB0E46E1:2057EB3B8A08ED65]:0)
[junit4] > at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:187)
[junit4] > at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:144)
[junit4] > at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
[junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:865)
[junit4] > at org.apache.solr.cloud.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:437)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 3439392 INFO (SUITE-ShardSplitTest-seed#[AB7038EACB0E46E1]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_AB7038EACB0E46E1-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {shard_s=PostingsFormat(name=LuceneVarGapDocFreqInterval), rnd_b=PostingsFormat(name=Memory doPackFST= false), n_ti=PostingsFormat(name=LuceneVarGapDocFreqInterval), a_t=PostingsFormat(name=LuceneVarGapDocFreqInterval), a_i=PostingsFormat(name=Memory doPackFST= false), id=PostingsFormat(name=Memory doPackFST= false)}, docValues:{shard_s=DocValuesFormat(name=Lucene54), rnd_b=DocValuesFormat(name=Direct), _version_=DocValuesFormat(name=Asserting), n_ti=DocValuesFormat(name=Lucene54), a_t=DocValuesFormat(name=Lucene54), a_i=DocValuesFormat(name=Direct), id=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=244, maxMBSortInHeap=7.316244207019235, sim=RandomSimilarity(queryNorm=true,coord=no): {}, locale=it, timezone=Africa/Ouagadougou
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=287076072,total=527433728
[junit4] 2> NOTE: All tests run in this JVM: [TestTrackingShardHandlerFactory, SecurityConfHandlerTest, TestIBSimilarityFactory, SuggestComponentTest, DeleteLastCustomShardedReplicaTest, TestInfoStreamLogging, CurrencyFieldXmlFileTest, TestLRUStatsCache, TestIntervalFaceting, HdfsSyncSliceTest, TestUpdate, DOMUtilTest, TestJavabinTupleStreamParser, TestDocBasedVersionConstraints, TestSSLRandomization, DistributedFacetPivotSmallAdvancedTest, TestEmbeddedSolrServerConstructors, BlockJoinFacetRandomTest, TestSlowCompositeReaderWrapper, TestStressRecovery, ResponseHeaderTest, TestCustomDocTransformer, TestSolrCoreParser, TestCSVResponseWriter, RollingRestartTest, TestPseudoReturnFields, TestFunctionQuery, ZkCLITest, AnalysisAfterCoreReloadTest, TestRestoreCore, TestDistributedSearch, TestSolrCloudWithSecureImpersonation, TestFieldCacheSort, TestUseDocValuesAsStored, TestFastLRUCache, ResponseLogComponentTest, TestSchemaNameResource, TestHdfsUpdateLog, TestLazyCores, ChangedSchemaMergeTest, SOLR749Test, TestSurroundQueryParser, TestDeleteCollectionOnDownNodes, TestCSVLoader, SubstringBytesRefFilterTest, TestNumericTerms64, TestShortCircuitedRequests, TestCursorMarkWithoutUniqueKey, DocValuesNotIndexedTest, TestSolrConfigHandlerConcurrent, HdfsRecoveryZkTest, DistributedFacetPivotWhiteBoxTest, OverriddenZkACLAndCredentialsProvidersTest, TestClusterStateMutator, TestNoOpRegenerator, DirectSolrSpellCheckerTest, SchemaVersionSpecificBehaviorTest, QueryElevationComponentTest, TestSubQueryTransformerDistrib, TestTrieFacet, TestLFUCache, TestWordDelimiterFilterFactory, MoreLikeThisHandlerTest, PeerSyncTest, AtomicUpdatesTest, TestSmileRequest, StatsReloadRaceTest, SharedFSAutoReplicaFailoverTest, TestBulkSchemaAPI, TestHdfsCloudBackupRestore, TestSQLHandlerNonCloud, TestAddFieldRealTimeGet, RecoveryZkTest, ManagedSchemaRoundRobinCloudTest, TestStressReorder, HdfsDirectoryTest, LeaderElectionTest, TestManagedResourceStorage, TestShardHandlerFactory, TestSolrJ, TestInPlaceUpdatesStandalone, HighlighterTest, XsltUpdateRequestHandlerTest, QueryEqualityTest, TestManagedStopFilterFactory, DistributedExpandComponentTest, RequestLoggingTest, InfoHandlerTest, TestStandardQParsers, TestSort, CursorPagingTest, TestPostingsSolrHighlighter, TestCollectionAPIs, TestComponentsName, HttpPartitionTest, TestLocalFSCloudBackupRestore, CdcrReplicationDistributedZkTest, DateFieldTest, GraphQueryTest, TestManagedSchema, DistributedSpellCheckComponentTest, TestRuleBasedAuthorizationPlugin, DistributedQueryComponentOptimizationTest, SpatialHeatmapFacetsTest, DistribDocExpirationUpdateProcessorTest, TestXIncludeConfig, SynonymTokenizerTest, DocValuesMultiTest, TestRequestForwarding, TestAuthenticationFramework, OverseerModifyCollectionTest, HdfsBasicDistributedZkTest, ZkNodePropsTest, ClusterStateUpdateTest, TestPerFieldSimilarity, CloudMLTQParserTest, ZkControllerTest, TestSubQueryTransformer, TestCloudManagedSchema, TestSolrCloudWithHadoopAuthPlugin, TestCloudInspectUtil, BaseCdcrDistributedZkTest, TestStressInPlaceUpdates, TestNRTOpen, TestOrdValues, TestConfigOverlay, PluginInfoTest, TestCorePropertiesReload, BigEndianAscendingWordSerializerTest, CoreSorterTest, DistributedMLTComponentTest, TestSerializedLuceneMatchVersion, PeerSyncReplicationTest, ConnectionManagerTest, MigrateRouteKeyTest, TestFaceting, TestElisionMultitermQuery, TestDownShardTolerantSearch, SolrMetricManagerTest, TestNonDefinedSimilarityFactory, TestClassicSimilarityFactory, SimpleFacetsTest, HdfsTlogReplayBufferedWhileIndexingTest, BasicDistributedZkTest, TestSolrCLIRunExample, DocValuesTest, TestSweetSpotSimilarityFactory, TestObjectReleaseTracker, TestDelegationWithHadoopAuth, TestSolrQueryParserResource, ExitableDirectoryReaderTest, TestBM25SimilarityFactory, SystemInfoHandlerTest, TlogReplayBufferedWhileIndexingTest, V2ApiIntegrationTest, PreAnalyzedUpdateProcessorTest, AnalyticsMergeStrategyTest, TestRawResponseWriter, TestCoreContainer, TestMiniSolrCloudClusterSSL, TestJmxIntegration, TestExpandComponent, FacetPivotSmallTest, TestSearcherReuse, TestFileDictionaryLookup, TestClusterProperties, TestAnalyzedSuggestions, TestGeoJSONResponseWriter, CustomCollectionTest, SparseHLLTest, TestSolrDeletionPolicy1, SyncSliceTest, LeaderElectionIntegrationTest, ShardRoutingTest, TestRandomFaceting, TestZkChroot, TestRandomDVFaceting, ShardRoutingCustomTest, AutoCommitTest, TestRecovery, TestRealTimeGet, TestReload, HardAutoCommitTest, SolrCoreTest, TestGroupingSearch, SolrCmdDistributorTest, DirectUpdateHandlerTest, SoftAutoCommitTest, TestRemoteStreaming, DebugComponentTest, IndexBasedSpellCheckerTest, StandardRequestHandlerTest, TestQueryTypes, TermsComponentTest, RequestHandlersTest, PathHierarchyTokenizerFactoryTest, FieldAnalysisRequestHandlerTest, FastVectorHighlighterTest, LoggingHandlerTest, SolrPluginUtilsTest, JSONWriterTest, QueryParsingTest, BinaryUpdateRequestHandlerTest, TestPHPSerializedResponseWriter, TestBinaryResponseWriter, HighlighterConfigTest, TestQuerySenderListener, UpdateParamsTest, AlternateDirectoryTest, TestSolrIndexConfig, TestQuerySenderNoQuery, CopyFieldTest, BadComponentTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, SampleTest, TestSearchPerf, OutputWriterTest, TestPhraseSuggestions, TestDFRSimilarityFactory, TimeZoneUtilsTest, ResourceLoaderTest, URLClassifyProcessorTest, ChaosMonkeyNothingIsSafeTest, CleanupOldIndexTest, CollectionStateFormat2Test, CollectionsAPIAsyncDistributedZkTest, DeleteStatusTest, DistribCursorPagingTest, DistributedVersionInfoTest, LeaderFailureAfterFreshStartTest, MultiThreadedOCPTest, RecoveryAfterSoftCommitTest, ShardSplitTest]
[junit4] Completed [550/698 (1!)] on J0 in 704.40s, 5 tests, 1 failure <<< FAILURES!
[...truncated 63188 lines...]