You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2017/10/20 11:42:57 UTC
[JENKINS] Lucene-Solr-Tests-7.x - Build # 188 - Still Unstable
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/188/
7 tests failed.
FAILED: org.apache.solr.cloud.HttpPartitionTest.test
Error Message:
Could not load collection from ZK: c8n_1x2
Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: c8n_1x2
at __randomizedtesting.SeedInfo.seed([5D3C760E1FD07E28:D56849D4B12C13D0]:0)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1963)
at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:370)
at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/c8n_1x2/state.json
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
... 46 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestCloudRecovery
Error Message:
Error from server at https://127.0.0.1:34391/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000000
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:34391/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000000
at __randomizedtesting.SeedInfo.seed([5D3C760E1FD07E28]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.TestCloudRecovery.setupCluster(TestCloudRecovery.java:68)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.SearchRateTriggerTest.testTrigger
Error Message:
expected:<0> but was:<1>
Stack Trace:
java.lang.AssertionError: expected:<0> but was:<1>
at __randomizedtesting.SeedInfo.seed([5D3C760E1FD07E28:3EF7408C861F0D05]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.autoscaling.SearchRateTriggerTest.testTrigger(SearchRateTriggerTest.java:101)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.update.processor.TestNamedUpdateProcessors.test
Error Message:
{ "responseHeader":{ "status":500, "QTime":31981}, "errorMessages":["4 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [https://127.0.0.1:53437/collection1_shard2_replica_n41/, https://127.0.0.1:44108/collection1_shard2_replica_n45/, https://127.0.0.1:59937/collection1_shard1_replica_n43/, https://127.0.0.1:36887/collection1_shard1_replica_n47/]\n"], "WARNING":"This response format is experimental. It is likely to change in the future.", "error":{ "metadata":[ "error-class","org.apache.solr.common.SolrException", "root-error-class","org.apache.solr.common.SolrException"], "msg":"4 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [https://127.0.0.1:53437/collection1_shard2_replica_n41/, https://127.0.0.1:44108/collection1_shard2_replica_n45/, https://127.0.0.1:59937/collection1_shard1_replica_n43/, https://127.0.0.1:36887/collection1_shard1_replica_n47/]", "trace":"org.apache.solr.common.SolrException: 4 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [https://127.0.0.1:53437/collection1_shard2_replica_n41/, https://127.0.0.1:44108/collection1_shard2_replica_n45/, https://127.0.0.1:59937/collection1_shard1_replica_n43/, https://127.0.0.1:36887/collection1_shard1_replica_n47/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:772)\n\tat org.apache.solr.handler.SolrConfigHandler.access$400(SolrConfigHandler.java:102)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:503)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:346)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.access$100(SolrConfigHandler.java:159)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:137)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2484)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:710)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:516)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:534)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)\n\tat java.lang.Thread.run(Thread.java:748)\n", "code":500}}
Stack Trace:
java.lang.AssertionError: {
"responseHeader":{
"status":500,
"QTime":31981},
"errorMessages":["4 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [https://127.0.0.1:53437/collection1_shard2_replica_n41/, https://127.0.0.1:44108/collection1_shard2_replica_n45/, https://127.0.0.1:59937/collection1_shard1_replica_n43/, https://127.0.0.1:36887/collection1_shard1_replica_n47/]\n"],
"WARNING":"This response format is experimental. It is likely to change in the future.",
"error":{
"metadata":[
"error-class","org.apache.solr.common.SolrException",
"root-error-class","org.apache.solr.common.SolrException"],
"msg":"4 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [https://127.0.0.1:53437/collection1_shard2_replica_n41/, https://127.0.0.1:44108/collection1_shard2_replica_n45/, https://127.0.0.1:59937/collection1_shard1_replica_n43/, https://127.0.0.1:36887/collection1_shard1_replica_n47/]",
"trace":"org.apache.solr.common.SolrException: 4 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [https://127.0.0.1:53437/collection1_shard2_replica_n41/, https://127.0.0.1:44108/collection1_shard2_replica_n45/, https://127.0.0.1:59937/collection1_shard1_replica_n43/, https://127.0.0.1:36887/collection1_shard1_replica_n47/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:772)\n\tat org.apache.solr.handler.SolrConfigHandler.access$400(SolrConfigHandler.java:102)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:503)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:346)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.access$100(SolrConfigHandler.java:159)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:137)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2484)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:710)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:516)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:534)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)\n\tat java.lang.Thread.run(Thread.java:748)\n",
"code":500}}
at __randomizedtesting.SeedInfo.seed([5D3C760E1FD07E28:D56849D4B12C13D0]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.core.TestSolrConfigHandler.runConfigCommand(TestSolrConfigHandler.java:185)
at org.apache.solr.update.processor.TestNamedUpdateProcessors.test(TestNamedUpdateProcessors.java:85)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI
Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:52419/solr/awhollynewcollection_0, http://127.0.0.1:59254/solr/awhollynewcollection_0, http://127.0.0.1:59800/solr/awhollynewcollection_0, http://127.0.0.1:45558/solr/awhollynewcollection_0]
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:52419/solr/awhollynewcollection_0, http://127.0.0.1:59254/solr/awhollynewcollection_0, http://127.0.0.1:59800/solr/awhollynewcollection_0, http://127.0.0.1:45558/solr/awhollynewcollection_0]
at __randomizedtesting.SeedInfo.seed([5D3C760E1FD07E28:154902BA19E351BD]:0)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:462)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:460)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:59254/solr/awhollynewcollection_0: no servers hosting shard: shard5
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
... 45 more
FAILED: org.apache.solr.schema.TestCloudSchemaless.test
Error Message:
Error from server at http://127.0.0.1:43259/jglt/b/collection1: 4 Async exceptions during distributed update: Error from server at http://127.0.0.1:60536/jglt/b/collection1_shard1_replica_n87: Server Error request: http://127.0.0.1:60536/jglt/b/collection1_shard1_replica_n87/update?update.chain=add-unknown-fields-to-the-schema&update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A43259%2Fjglt%2Fb%2Fcollection1_shard3_replica_n91%2F&wt=javabin&version=2 Remote error message: Error persisting managed schema at /configs/conf1/managed-schema Error from server at http://127.0.0.1:43224/jglt/b/collection1_shard4_replica_n85: Server Error request: http://127.0.0.1:43224/jglt/b/collection1_shard4_replica_n85/update?update.chain=add-unknown-fields-to-the-schema&update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A43259%2Fjglt%2Fb%2Fcollection1_shard3_replica_n91%2F&wt=javabin&version=2 Remote error message: Error persisting managed schema at /configs/conf1/managed-schema Error from server at http://127.0.0.1:37445/jglt/b/collection1_shard3_replica_n83: Server Error request: http://127.0.0.1:37445/jglt/b/collection1_shard3_replica_n83/update?update.chain=add-unknown-fields-to-the-schema&update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A43259%2Fjglt%2Fb%2Fcollection1_shard3_replica_n91%2F&wt=javabin&version=2 Remote error message: Error persisting managed schema at /configs/conf1/managed-schema Error from server at http://127.0.0.1:51945/jglt/b/collection1_shard2_replica_n81: Server Error request: http://127.0.0.1:51945/jglt/b/collection1_shard2_replica_n81/update?update.chain=add-unknown-fields-to-the-schema&update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A43259%2Fjglt%2Fb%2Fcollection1_shard3_replica_n91%2F&wt=javabin&version=2 Remote error message: Error persisting managed schema at /configs/conf1/managed-schema
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:43259/jglt/b/collection1: 4 Async exceptions during distributed update:
Error from server at http://127.0.0.1:60536/jglt/b/collection1_shard1_replica_n87: Server Error
request: http://127.0.0.1:60536/jglt/b/collection1_shard1_replica_n87/update?update.chain=add-unknown-fields-to-the-schema&update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A43259%2Fjglt%2Fb%2Fcollection1_shard3_replica_n91%2F&wt=javabin&version=2
Remote error message: Error persisting managed schema at /configs/conf1/managed-schema
Error from server at http://127.0.0.1:43224/jglt/b/collection1_shard4_replica_n85: Server Error
request: http://127.0.0.1:43224/jglt/b/collection1_shard4_replica_n85/update?update.chain=add-unknown-fields-to-the-schema&update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A43259%2Fjglt%2Fb%2Fcollection1_shard3_replica_n91%2F&wt=javabin&version=2
Remote error message: Error persisting managed schema at /configs/conf1/managed-schema
Error from server at http://127.0.0.1:37445/jglt/b/collection1_shard3_replica_n83: Server Error
request: http://127.0.0.1:37445/jglt/b/collection1_shard3_replica_n83/update?update.chain=add-unknown-fields-to-the-schema&update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A43259%2Fjglt%2Fb%2Fcollection1_shard3_replica_n91%2F&wt=javabin&version=2
Remote error message: Error persisting managed schema at /configs/conf1/managed-schema
Error from server at http://127.0.0.1:51945/jglt/b/collection1_shard2_replica_n81: Server Error
request: http://127.0.0.1:51945/jglt/b/collection1_shard2_replica_n81/update?update.chain=add-unknown-fields-to-the-schema&update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A43259%2Fjglt%2Fb%2Fcollection1_shard3_replica_n91%2F&wt=javabin&version=2
Remote error message: Error persisting managed schema at /configs/conf1/managed-schema
at __randomizedtesting.SeedInfo.seed([5D3C760E1FD07E28:D56849D4B12C13D0]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:106)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:71)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:85)
at org.apache.solr.schema.TestCloudSchemaless.test(TestCloudSchemaless.java:124)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit
Error Message:
Shard split did not succeed after a previous failed split attempt left sub-shards in construction state
Stack Trace:
java.lang.AssertionError: Shard split did not succeed after a previous failed split attempt left sub-shards in construction state
at __randomizedtesting.SeedInfo.seed([5D3C760E1FD07E28:A471E5A123A533A2]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:294)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12238 lines...]
[junit4] Suite: org.apache.solr.cloud.TestCloudRecovery
[junit4] 2> 2538711 INFO (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/init-core-data-001
[junit4] 2> 2538712 WARN (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=21 numCloses=21
[junit4] 2> 2538712 INFO (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 2538714 INFO (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 2538723 INFO (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001
[junit4] 2> 2538724 INFO (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2539001 INFO (Thread-1492) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2539001 INFO (Thread-1492) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2539114 ERROR (Thread-1492) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2539114 INFO (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:41174
[junit4] 2> 2539502 INFO (jetty-launcher-838-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2539513 INFO (jetty-launcher-838-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2539514 INFO (jetty-launcher-838-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@45f94c23{/solr,null,AVAILABLE}
[junit4] 2> 2539535 INFO (jetty-launcher-838-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1c94313d{SSL,[ssl, http/1.1]}{127.0.0.1:34391}
[junit4] 2> 2539535 INFO (jetty-launcher-838-thread-2) [ ] o.e.j.s.Server Started @2557456ms
[junit4] 2> 2539535 INFO (jetty-launcher-838-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34391}
[junit4] 2> 2539535 ERROR (jetty-launcher-838-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2539535 INFO (jetty-launcher-838-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 2539535 INFO (jetty-launcher-838-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2539535 INFO (jetty-launcher-838-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2539535 INFO (jetty-launcher-838-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-20T08:00:30.166Z
[junit4] 2> 2539538 INFO (jetty-launcher-838-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3648a272{/solr,null,AVAILABLE}
[junit4] 2> 2539538 INFO (jetty-launcher-838-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@69ceb7c8{SSL,[ssl, http/1.1]}{127.0.0.1:47185}
[junit4] 2> 2539538 INFO (jetty-launcher-838-thread-1) [ ] o.e.j.s.Server Started @2557459ms
[junit4] 2> 2539538 INFO (jetty-launcher-838-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=47185}
[junit4] 2> 2539538 ERROR (jetty-launcher-838-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2539538 INFO (jetty-launcher-838-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 2539538 INFO (jetty-launcher-838-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2539538 INFO (jetty-launcher-838-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2539538 INFO (jetty-launcher-838-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-20T08:00:30.169Z
[junit4] 2> 2539558 INFO (jetty-launcher-838-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2539632 INFO (jetty-launcher-838-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2539922 INFO (jetty-launcher-838-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41174/solr
[junit4] 2> 2540624 INFO (jetty-launcher-838-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41174/solr
[junit4] 2> 2540910 INFO (jetty-launcher-838-thread-1) [n:127.0.0.1:47185_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2540910 INFO (jetty-launcher-838-thread-1) [n:127.0.0.1:47185_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:47185_solr
[junit4] 2> 2540911 INFO (jetty-launcher-838-thread-1) [n:127.0.0.1:47185_solr ] o.a.s.c.Overseer Overseer (id=98860166652624902-127.0.0.1:47185_solr-n_0000000000) starting
[junit4] 2> 2541023 INFO (jetty-launcher-838-thread-1) [n:127.0.0.1:47185_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47185_solr
[junit4] 2> 2541177 INFO (OverseerStateUpdate-98860166652624902-127.0.0.1:47185_solr-n_0000000000) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2541910 INFO (jetty-launcher-838-thread-2) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2541911 INFO (jetty-launcher-838-thread-2) [n:127.0.0.1:34391_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2541942 INFO (jetty-launcher-838-thread-2) [n:127.0.0.1:34391_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34391_solr
[junit4] 2> 2541943 INFO (zkCallback-851-thread-1-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2542250 INFO (zkCallback-849-thread-1-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2543120 INFO (jetty-launcher-838-thread-2) [n:127.0.0.1:34391_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34391.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2543245 INFO (jetty-launcher-838-thread-1) [n:127.0.0.1:47185_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47185.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2543354 INFO (jetty-launcher-838-thread-2) [n:127.0.0.1:34391_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34391.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2543451 INFO (jetty-launcher-838-thread-2) [n:127.0.0.1:34391_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34391.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2543452 INFO (jetty-launcher-838-thread-1) [n:127.0.0.1:47185_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47185.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2543452 INFO (jetty-launcher-838-thread-1) [n:127.0.0.1:47185_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47185.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2543467 INFO (jetty-launcher-838-thread-2) [n:127.0.0.1:34391_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001/node2/.
[junit4] 2> 2543467 INFO (jetty-launcher-838-thread-1) [n:127.0.0.1:47185_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001/node1/.
[junit4] 2> 2544082 INFO (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2544082 INFO (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41174/solr ready
[junit4] 2> 2544378 INFO (qtp698573975-5431) [n:127.0.0.1:34391_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=0&collection.configName=config&maxShardsPerNode=2&name=collection1&nrtReplicas=0&action=CREATE&numShards=2&tlogReplicas=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2544433 INFO (OverseerThreadFactory-2191-thread-1-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 2544647 INFO (OverseerStateUpdate-98860166652624902-127.0.0.1:47185_solr-n_0000000000) [n:127.0.0.1:47185_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"collection1_shard1_replica_t1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:34391/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2544650 INFO (OverseerStateUpdate-98860166652624902-127.0.0.1:47185_solr-n_0000000000) [n:127.0.0.1:47185_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"collection1_shard1_replica_t3",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:47185/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2544670 INFO (OverseerStateUpdate-98860166652624902-127.0.0.1:47185_solr-n_0000000000) [n:127.0.0.1:47185_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"collection1_shard2_replica_t5",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:34391/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2544672 INFO (OverseerStateUpdate-98860166652624902-127.0.0.1:47185_solr-n_0000000000) [n:127.0.0.1:47185_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"collection1_shard2_replica_t7",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:47185/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2545200 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 2545201 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2545235 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t7&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 2545235 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2545816 INFO (zkCallback-849-thread-1-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2545816 INFO (zkCallback-851-thread-1-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2545819 INFO (zkCallback-851-thread-2-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2545856 INFO (zkCallback-849-thread-2-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2545902 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t3&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 2545922 INFO (qtp698573975-5425) [n:127.0.0.1:34391_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node6&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t5&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 2546085 INFO (zkCallback-849-thread-2-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2546085 INFO (zkCallback-849-thread-1-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2546086 INFO (zkCallback-851-thread-3-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2546086 INFO (zkCallback-851-thread-2-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2546777 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2546812 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2565560 WARN (jetty-launcher-838-thread-2-SendThread(127.0.0.1:41174)) [n:127.0.0.1:34391_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 18626ms for sessionid 0x15f38cc3bc40005
[junit4] 2> 2565568 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.s.IndexSchema [collection1_shard2_replica_t7] Schema name=minimal
[junit4] 2> 2565589 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2565589 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t7' using configuration from collection collection1, trusted=true
[junit4] 2> 2565590 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47185.solr.core.collection1.shard2.replica_t7' (registry 'solr.core.collection1.shard2.replica_t7') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2565590 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2565590 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.c.SolrCore [[collection1_shard2_replica_t7] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001/node1/collection1_shard2_replica_t7], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001/node1/./collection1_shard2_replica_t7/data/]
[junit4] 2> 2565560 WARN (jetty-launcher-838-thread-1-SendThread(127.0.0.1:41174)) [n:127.0.0.1:47185_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 18636ms for sessionid 0x15f38cc3bc40006
[junit4] 2> 2565765 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f38cc3bc40006, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2565794 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f38cc3bc40005, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2565678 WARN (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3187160e name: ZooKeeperConnection Watcher:127.0.0.1:41174/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 2565561 WARN (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker-SendThread(127.0.0.1:41174)) [ ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 21119ms for sessionid 0x15f38cc3bc40008
[junit4] 2> 2565794 WARN (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 2565918 WARN (zkCallback-851-thread-2-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@103cfaa5 name: ZooKeeperConnection Watcher:127.0.0.1:41174/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 2565918 WARN (zkCallback-851-thread-2-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 2565921 WARN (zkCallback-857-thread-2) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7e307391 name: ZooKeeperConnection Watcher:127.0.0.1:41174/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 2565921 WARN (zkCallback-857-thread-2) [ ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 2566357 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2566357 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2566359 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2566359 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2566397 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.s.SolrIndexSearcher Opening [Searcher@538b0dd1[collection1_shard2_replica_t7] main]
[junit4] 2> 2567036 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
[junit4] 2> 2567037 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
[junit4] 2> 2567038 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2567038 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581762695836729344
[junit4] 2> 2567078 INFO (searcherExecutor-2196-thread-1-processing-n:127.0.0.1:47185_solr x:collection1_shard2_replica_t7 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.c.SolrCore [collection1_shard2_replica_t7] Registered new searcher Searcher@538b0dd1[collection1_shard2_replica_t7] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2567106 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=9999ms
[junit4] 2> 2567129 INFO (zkCallback-851-thread-8-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2567141 INFO (zkCallback-851-thread-7-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2567397 INFO (zkCallback-851-thread-7-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 2567410 WARN (jetty-launcher-838-thread-2-SendThread(127.0.0.1:41174)) [n:127.0.0.1:34391_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15f38cc3bc40005 has expired
[junit4] 2> 2567413 WARN (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3187160e name: ZooKeeperConnection Watcher:127.0.0.1:41174/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
[junit4] 2> 2567413 WARN (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
[junit4] 2> 2567413 WARN (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
[junit4] 2> 2567425 INFO (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
[junit4] 2> 2567425 INFO (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
[junit4] 2> 2567457 INFO (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 2567463 WARN (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker-SendThread(127.0.0.1:41174)) [ ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15f38cc3bc40008 has expired
[junit4] 2> 2567464 WARN (zkCallback-857-thread-4) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7e307391 name: ZooKeeperConnection Watcher:127.0.0.1:41174/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
[junit4] 2> 2567464 WARN (zkCallback-857-thread-4) [ ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
[junit4] 2> 2567464 WARN (zkCallback-857-thread-4) [ ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
[junit4] 2> 2567502 INFO (zkCallback-857-thread-4) [ ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
[junit4] 2> 2567517 ERROR (qtp698573975-5425) [n:127.0.0.1:34391_solr c:collection1 s:shard2 r:core_node6 x:collection1_shard2_replica_t5] o.a.s.c.CloudConfigSetService null:org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
[junit4] 2> at org.apache.solr.cloud.CloudConfigSetService.createCoreResourceLoader(CloudConfigSetService.java:44)
[junit4] 2> at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:75)
[junit4] 2> at org.apache.solr.core.CoreContainer.getConfigSet(CoreContainer.java:1080)
[junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1032)
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:948)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2>
[junit4] 2> 2567527 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2567569 INFO (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34391_solr
[junit4] 2> 2567593 INFO (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnected to ZooKeeper
[junit4] 2> 2567593 INFO (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ConnectionManager Connected:true
[junit4] 2> 2567612 INFO (qtp698573975-5425) [n:127.0.0.1:34391_solr c:collection1 s:shard2 r:core_node6 x:collection1_shard2_replica_t5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2567650 INFO (zkCallback-851-thread-7-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2567666 INFO (zkCallback-857-thread-4) [ ] o.a.s.c.c.DefaultConnectionStrategy Reconnected to ZooKeeper
[junit4] 2> 2567666 INFO (zkCallback-857-thread-4) [ ] o.a.s.c.c.ConnectionManager Connected:true
[junit4] 2> 2567689 INFO (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2567695 INFO (qtp698573975-5425) [n:127.0.0.1:34391_solr c:collection1 s:shard2 r:core_node6 x:collection1_shard2_replica_t5] o.a.s.s.IndexSchema [collection1_shard2_replica_t5] Schema name=minimal
[junit4] 2> 2567751 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.s.IndexSchema [collection1_shard1_replica_t3] Schema name=minimal
[junit4] 2> 2567765 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2567765 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t3' using configuration from collection collection1, trusted=true
[junit4] 2> 2567766 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47185.solr.core.collection1.shard1.replica_t3' (registry 'solr.core.collection1.shard1.replica_t3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2567766 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2567766 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.c.SolrCore [[collection1_shard1_replica_t3] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001/node1/collection1_shard1_replica_t3], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001/node1/./collection1_shard1_replica_t3/data/]
[junit4] 2> 2567785 INFO (zkCallback-851-thread-7-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2567786 INFO (zkCallback-851-thread-8-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2567786 INFO (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2567787 INFO (qtp698573975-5425) [n:127.0.0.1:34391_solr c:collection1 s:shard2 r:core_node6 x:collection1_shard2_replica_t5] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2567787 INFO (qtp698573975-5425) [n:127.0.0.1:34391_solr c:collection1 s:shard2 r:core_node6 x:collection1_shard2_replica_t5] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t5' using configuration from collection collection1, trusted=true
[junit4] 2> 2567788 INFO (qtp698573975-5425) [n:127.0.0.1:34391_solr c:collection1 s:shard2 r:core_node6 x:collection1_shard2_replica_t5] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34391.solr.core.collection1.shard2.replica_t5' (registry 'solr.core.collection1.shard2.replica_t5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2567788 INFO (qtp698573975-5425) [n:127.0.0.1:34391_solr c:collection1 s:shard2 r:core_node6 x:collection1_shard2_replica_t5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2567788 INFO (qtp698573975-5425) [n:127.0.0.1:34391_solr c:collection1 s:shard2 r:core_node6 x:collection1_shard2_replica_t5] o.a.s.c.SolrCore [[collection1_shard2_replica_t5] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001/node2/collection1_shard2_replica_t5], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001/node2/./collection1_shard2_replica_t5/data/]
[junit4] 2> 2568379 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2568379 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2568386 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2568386 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2568432 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.s.SolrIndexSearcher Opening [Searcher@3c9e875e[collection1_shard1_replica_t3] main]
[junit4] 2> 2568556 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.s.IndexSchema [collection1_shard1_replica_t1] Schema name=minimal
[junit4] 2> 2568580 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2568580 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t1' using configuration from collection collection1, trusted=true
[junit4] 2> 2568581 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34391.solr.core.collection1.shard1.replica_t1' (registry 'solr.core.collection1.shard1.replica_t1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2568581 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2568593 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr c:collection1 s:shard1 r:core_node2 x:collec
[...truncated too long message...]
2a3c387: rootName = solr_57520, domain = solr.core.SearchRateTriggerTest-collection1.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.SearchRateTriggerTest-collection1.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@18936c02
[junit4] 2> 9392383 INFO (coreCloseExecutor-8765-thread-2) [n:127.0.0.1:57520_solr c:SearchRateTriggerTest-collection2 s:shard2 r:core_node6 x:SearchRateTriggerTest-collection2_shard2_replica_n5] o.a.s.c.SolrCore [SearchRateTriggerTest-collection2_shard2_replica_n5] CLOSING SolrCore org.apache.solr.core.SolrCore@6a79cd06
[junit4] 2> 9392389 INFO (coreCloseExecutor-8765-thread-2) [n:127.0.0.1:57520_solr c:SearchRateTriggerTest-collection2 s:shard2 r:core_node6 x:SearchRateTriggerTest-collection2_shard2_replica_n5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.SearchRateTriggerTest-collection2.shard2.replica_n5, tag=1786367238
[junit4] 2> 9392389 INFO (coreCloseExecutor-8765-thread-2) [n:127.0.0.1:57520_solr c:SearchRateTriggerTest-collection2 s:shard2 r:core_node6 x:SearchRateTriggerTest-collection2_shard2_replica_n5] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4c790c85: rootName = solr_57520, domain = solr.core.SearchRateTriggerTest-collection2.shard2.replica_n5, service url = null, agent id = null] for registry solr.core.SearchRateTriggerTest-collection2.shard2.replica_n5 / com.codahale.metrics.MetricRegistry@36bb7d9f
[junit4] 2> 9392434 INFO (coreCloseExecutor-8765-thread-2) [n:127.0.0.1:57520_solr c:SearchRateTriggerTest-collection2 s:shard2 r:core_node6 x:SearchRateTriggerTest-collection2_shard2_replica_n5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.SearchRateTriggerTest-collection2.shard2.leader, tag=1786367238
[junit4] 2> 9392482 INFO (coreCloseExecutor-8765-thread-1) [n:127.0.0.1:57520_solr c:SearchRateTriggerTest-collection1 s:shard1 r:core_node5 x:SearchRateTriggerTest-collection1_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.SearchRateTriggerTest-collection1.shard1.leader, tag=30065930
[junit4] 2> 9392803 INFO (jetty-closer-4996-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@193ce9af{/solr,null,UNAVAILABLE}
[junit4] 2> 9392804 INFO (jetty-closer-4996-thread-2) [ ] o.a.s.c.Overseer Overseer (id=98860612339105800-127.0.0.1:54907_solr-n_0000000000) closing
[junit4] 2> 9392804 INFO (OverseerStateUpdate-98860612339105800-127.0.0.1:54907_solr-n_0000000000) [n:127.0.0.1:54907_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:54907_solr
[junit4] 2> 9392867 WARN (OverseerAutoScalingTriggerThread-98860612339105800-127.0.0.1:54907_solr-n_0000000000) [n:127.0.0.1:54907_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 9392954 INFO (jetty-closer-4996-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@17c31e63{/solr,null,UNAVAILABLE}
[junit4] 2> 9392968 INFO (zkCallback-5018-thread-8-processing-n:127.0.0.1:57520_solr) [n:127.0.0.1:57520_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:57520_solr
[junit4] 2> 9392969 INFO (jetty-closer-4996-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7d5539ed{/solr,null,UNAVAILABLE}
[junit4] 2> 9393031 INFO (jetty-closer-4996-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3538438d{/solr,null,UNAVAILABLE}
[junit4] 2> 9393068 ERROR (SUITE-SearchRateTriggerTest-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 9393068 INFO (SUITE-SearchRateTriggerTest-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:57676 57676
[junit4] 2> 9393158 INFO (Thread-4652) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:57676 57676
[junit4] 2> 9393184 WARN (Thread-4652) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 4 /solr/security.json
[junit4] 2> 4 /solr/configs/conf
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 53 /solr/collections/SearchRateTriggerTest-collection1/state.json
[junit4] 2> 48 /solr/collections/SearchRateTriggerTest-collection2/state.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 2 /solr/overseer_elect/election/98860612339105800-127.0.0.1:54907_solr-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/live_nodes
[junit4] 2> 6 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.SearchRateTriggerTest_5D3C760E1FD07E28-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1297, maxMBSortInHeap=6.6087307604771155, sim=RandomSimilarity(queryNorm=true): {}, locale=ar-SD, timezone=SystemV/MST7MDT
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=247663976,total=520617984
[junit4] 2> NOTE: All tests run in this JVM: [SuggesterTSTTest, ClusterStateUpdateTest, TestLegacyTerms, HdfsChaosMonkeySafeLeaderTest, NotRequiredUniqueKeyTest, SolrCmdDistributorTest, TestDynamicFieldResource, PrimUtilsTest, InfixSuggestersTest, ImplicitSnitchTest, SynonymTokenizerTest, MoreLikeThisHandlerTest, ExecutePlanActionTest, TestManagedResource, SharedFSAutoReplicaFailoverTest, SOLR749Test, SuggesterFSTTest, TestCollectionAPIs, TestSurroundQueryParser, BasicAuthStandaloneTest, TestReloadAndDeleteDocs, TestFastWriter, TestInitQParser, TestSimpleTrackingShardHandler, SimpleMLTQParserTest, TestDocumentBuilder, TestStressLiveNodes, TestSolrQueryParser, CdcrBootstrapTest, TestInPlaceUpdatesStandalone, NodeMutatorTest, ZkSolrClientTest, ShardRoutingTest, TestBlobHandler, TestDocSet, HdfsRecoverLeaseTest, TestReRankQParserPlugin, QueryEqualityTest, TestDelegationWithHadoopAuth, TestCloudDeleteByQuery, TestSubQueryTransformer, TestMacros, CloneFieldUpdateProcessorFactoryTest, TestFieldCacheSort, UninvertDocValuesMergePolicyTest, TestCollectionsAPIViaSolrCloudCluster, BlobRepositoryCloudTest, TemplateUpdateProcessorTest, TermVectorComponentDistributedTest, HdfsUnloadDistributedZkTest, TestManagedSchema, TestSchemaResource, TestRemoteStreaming, ChaosMonkeySafeLeaderTest, DistributedDebugComponentTest, TestSegmentSorting, DeleteShardTest, TestStressRecovery, SolrMetricManagerTest, ConvertedLegacyTest, OpenCloseCoreStressTest, SolrCloudExampleTest, RequestHandlersTest, TestSolrConfigHandlerConcurrent, TestMissingGroups, TestManagedSynonymFilterFactory, CollectionTooManyReplicasTest, TestStressLucene, TestSolrConfigHandler, AssignTest, TestGeoJSONResponseWriter, LeaderFailureAfterFreshStartTest, HighlighterMaxOffsetTest, TestFastLRUCache, MergeStrategyTest, BooleanFieldTest, DistributedVersionInfoTest, TestJettySolrRunner, ManagedSchemaRoundRobinCloudTest, DirectUpdateHandlerTest, SSLMigrationTest, NumericFieldsTest, TestDistribDocBasedVersion, MoveReplicaHDFSTest, TestRetrieveFieldsOptimizer, SolrGangliaReporterTest, TestDistributedStatsComponentCardinality, CoreAdminRequestStatusTest, CloudExitableDirectoryReaderTest, ChaosMonkeySafeLeaderWithPullReplicasTest, TestCollationField, TestScoreJoinQPScore, TestSolr4Spatial, TestAnalyzedSuggestions, SampleTest, CollectionsAPIDistributedZkTest, TestLazyCores, TestQueryWrapperFilter, NodeAddedTriggerTest, TestCloudSchemaless, TestSkipOverseerOperations, CdcrRequestHandlerTest, SpatialHeatmapFacetsTest, TestAtomicUpdateErrorCases, TestDefaultStatsCache, DistribDocExpirationUpdateProcessorTest, TestHdfsCloudBackupRestore, ConcurrentDeleteAndCreateCollectionTest, TestJmxIntegration, DistributedTermsComponentTest, TestPushWriter, SchemaVersionSpecificBehaviorTest, TestPullReplica, TestMiniSolrCloudClusterSSL, SpatialRPTFieldTypeTest, ExitableDirectoryReaderTest, TestPivotHelperCode, TestConfigSetsAPIZkFailure, OpenExchangeRatesOrgProviderTest, PolyFieldTest, LukeRequestHandlerTest, TestOmitPositions, PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, FieldAnalysisRequestHandlerTest, RegexBoostProcessorTest, ReturnFieldsTest, QueryParsingTest, TestPartialUpdateDeduplication, CSVRequestHandlerTest, TestComponentsName, HighlighterConfigTest, TestSolrIndexConfig, CopyFieldTest, ResponseLogComponentTest, BadComponentTest, TestSolrDeletionPolicy2, TestFuzzyAnalyzedSuggestions, DirectSolrConnectionTest, TestCodecSupport, FileUtilsTest, TestRTGBase, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestHighlightDedupGrouping, AliasIntegrationTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, DeleteLastCustomShardedReplicaTest, DeleteReplicaTest, LeaderInitiatedRecoveryOnCommitTest, MissingSegmentRecoveryTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerStatusTest, ReplicationFactorTest, RollingRestartTest, SolrXmlInZkTest, TestCloudPivotFacet, TestCollectionAPI, TestCryptoKeys, TestLeaderInitiatedRecoveryThread, TestPrepRecovery, TestSizeLimitedDistributedMap, TestSolrCloudWithDelegationTokens, TestStressCloudBlindAtomicUpdates, TestTolerantUpdateProcessorRandomCloud, AutoAddReplicasIntegrationTest, SearchRateTriggerTest]
[junit4] Completed [553/744 (7!)] on J1 in 56.50s, 1 test, 1 failure <<< FAILURES!
[...truncated 47921 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 202 - Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/202/
All tests passed
Build Log:
[...truncated 4708 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:826: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:770: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build.xml:495: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:2262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:409: impossible to resolve dependencies:
resolve failed - see output for details
Total time: 115 minutes 8 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 201 - Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/201/
All tests passed
Build Log:
[...truncated 4723 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:826: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:770: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build.xml:495: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:2262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:409: impossible to resolve dependencies:
resolve failed - see output for details
Total time: 117 minutes 26 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 200 - Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/200/
All tests passed
Build Log:
[...truncated 1364 lines...]
[junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5418177546657407218320.sysout
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] #
[junit4] # There is insufficient memory for the Java Runtime Environment to continue.
[junit4] # Native memory allocation (mmap) failed to map 41418752 bytes for committing reserved memory.
[junit4] # An error report file with more information is saved as:
[junit4] # /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/J1/hs_err_pid10311.log
[junit4] <<< JVM J1: EOF ----
[junit4] JVM J1: stderr was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5411283484380853938762.syserr
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000000ed080000, 41418752, 0) failed; error='Cannot allocate memory' (errno=12)
[junit4] <<< JVM J1: EOF ----
[...truncated 288 lines...]
[junit4] ERROR: JVM J1 ended with an exception, command line: /usr/local/asfpackages/java/jdk1.8.0_144/jre/bin/java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/heapdumps -ea -esa -Dtests.prefix=tests -Dtests.seed=AB360698341A6835 -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=europarl.lines.txt.gz -Dtests.luceneMatchVersion=7.2.0 -Dtests.cleanthreads=perMethod -Djava.util.logging.config.file=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/tools/junit4/logging.properties -Dtests.nightly=false -Dtests.weekly=false -Dtests.monster=false -Dtests.slow=true -Dtests.asserts=true -Dtests.multiplier=2 -DtempDir=./temp -Djava.io.tmpdir=./temp -Djunit4.tempDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp -Dcommon.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene -Dclover.db.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/clover/db -Djava.security.policy=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/tools/junit4/tests.policy -Dtests.LUCENE_VERSION=7.2.0 -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Dtests.src.home=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x -Djava.security.egd=file:/dev/./urandom -Djunit4.childvm.cwd=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/J1 -Djunit4.childvm.id=1 -Djunit4.childvm.count=3 -Dtests.leaveTemporary=false -Dtests.filterstacks=true -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -Dfile.encoding=ISO-8859-1 -classpath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/codecs/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/test-framework/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/test-framework/lib/junit-4.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/test-framework/lib/randomizedtesting-runner-2.5.3.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/classes/test:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_144/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5418151426299979181834.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5412074331131381671030.suites -stdin
[junit4] ERROR: JVM J1 ended with an exception: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5418177546657407218320.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5411283484380853938762.syserr
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1519)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
[junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] at java.lang.Thread.run(Thread.java:748)
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:826: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:770: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build.xml:50: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1494: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1021: At least one slave process threw an exception, first: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5418177546657407218320.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5411283484380853938762.syserr
Total time: 46 minutes 59 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-Tests-7.x #197 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 199 - Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/199/
1 tests failed.
FAILED: org.apache.solr.cloud.TestLocalFSCloudBackupRestore.test
Error Message:
expected:<COMPLETED> but was:<RUNNING>
Stack Trace:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>
at __randomizedtesting.SeedInfo.seed([39042BD03F00148:8BC47D67AD0C6CB0]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:277)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11578 lines...]
[junit4] Suite: org.apache.solr.cloud.TestLocalFSCloudBackupRestore
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/init-core-data-001
[junit4] 2> 0 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 704 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.e.j.u.log Logging initialized @26167ms
[junit4] 2> 801 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 886 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 1035 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001
[junit4] 2> 1153 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1156 INFO (Thread-1) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1156 INFO (Thread-1) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1884 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:35952
[junit4] 2> 2990 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
[junit4] 2> 3627 ERROR (Thread-1) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 5182 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f7083c0440000, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 9513 INFO (jetty-launcher-1-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 9549 INFO (jetty-launcher-1-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 9930 INFO (jetty-launcher-1-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@34e710a4{/solr,null,AVAILABLE}
[junit4] 2> 9964 INFO (jetty-launcher-1-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7db77681{/solr,null,AVAILABLE}
[junit4] 2> 10051 INFO (jetty-launcher-1-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@39e9b53{HTTP/1.1,[http/1.1]}{127.0.0.1:41201}
[junit4] 2> 10052 INFO (jetty-launcher-1-thread-2) [ ] o.e.j.s.Server Started @35612ms
[junit4] 2> 10052 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41201}
[junit4] 2> 10055 INFO (jetty-launcher-1-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5a6ff595{HTTP/1.1,[http/1.1]}{127.0.0.1:58560}
[junit4] 2> 10056 INFO (jetty-launcher-1-thread-1) [ ] o.e.j.s.Server Started @35616ms
[junit4] 2> 10056 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58560}
[junit4] 2> 10371 ERROR (jetty-launcher-1-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 10371 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 10375 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 10375 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 10376 ERROR (jetty-launcher-1-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 10377 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 10428 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 10428 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 10487 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-31T03:40:11.947Z
[junit4] 2> 10551 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-31T03:40:11.911Z
[junit4] 2> 10667 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
[junit4] 2> 10972 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 11026 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 18546 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35952/solr
[junit4] 2> 18551 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35952/solr
[junit4] 2> 18680 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f7083c0440004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 20262 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 20270 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:41201_solr
[junit4] 2> 20280 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 20303 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.c.Overseer Overseer (id=98921427994148869-127.0.0.1:41201_solr-n_0000000000) starting
[junit4] 2> 20307 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58560_solr
[junit4] 2> 20378 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 20379 INFO (zkCallback-12-thread-1-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 21700 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41201_solr
[junit4] 2> 21757 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 21763 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 27189 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27221 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27222 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27466 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27500 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27500 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27692 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/.
[junit4] 2> 27764 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/.
[junit4] 2> 29970 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 29972 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35952/solr ready
[junit4] 2> 30097 INFO (TEST-TestLocalFSCloudBackupRestore.test-seed#[39042BD03F00148]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 33822 INFO (qtp1422851308-35) [n:127.0.0.1:58560_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&router.name=implicit&version=2&pullReplicas=1&shards=shard1,shard2&property.customKey=customValue&maxShardsPerNode=3&router.field=shard_s&autoAddReplicas=true&name=backuprestore&nrtReplicas=1&action=CREATE&tlogReplicas=1&wt=javabin and sendToOCPQueue=true
[junit4] 2> 34064 INFO (OverseerThreadFactory-12-thread-1-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.CreateCollectionCmd Create collection backuprestore
[junit4] 2> 34070 WARN (OverseerThreadFactory-12-thread-1-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 3 on collection backuprestore is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
[junit4] 2> 34296 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"backuprestore_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:58560/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 34534 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"backuprestore_shard1_replica_t2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:41201/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 34536 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"backuprestore_shard1_replica_p3",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:58560/solr",
[junit4] 2> "type":"PULL",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 34552 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"backuprestore_shard2_replica_n4",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:41201/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 34553 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"backuprestore_shard2_replica_t5",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:58560/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 34572 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"backuprestore_shard2_replica_p6",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:41201/solr",
[junit4] 2> "type":"PULL",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 35287 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node11&name=backuprestore_shard2_replica_t5&action=CREATE&numShards=2&shard=shard2&wt=javabin
[junit4] 2> 35287 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node10&name=backuprestore_shard2_replica_n4&action=CREATE&numShards=2&shard=shard2&wt=javabin
[junit4] 2> 35325 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node9&name=backuprestore_shard1_replica_p3&action=CREATE&numShards=2&shard=shard1&wt=javabin
[junit4] 2> 35326 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node7&name=backuprestore_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&wt=javabin
[junit4] 2> 35329 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node8&name=backuprestore_shard1_replica_t2&action=CREATE&numShards=2&shard=shard1&wt=javabin
[junit4] 2> 35388 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 35390 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 35416 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node12&name=backuprestore_shard2_replica_p6&action=CREATE&numShards=2&shard=shard2&wt=javabin
[junit4] 2> 35721 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 35727 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 36801 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 36843 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 38736 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 38756 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 38770 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.IndexSchema [backuprestore_shard2_replica_p6] Schema name=minimal
[junit4] 2> 38776 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 38820 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 38848 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.IndexSchema [backuprestore_shard1_replica_t2] Schema name=minimal
[junit4] 2> 40270 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.IndexSchema [backuprestore_shard1_replica_p3] Schema name=minimal
[junit4] 2> 40279 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.s.IndexSchema [backuprestore_shard2_replica_t5] Schema name=minimal
[junit4] 2> 40284 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.s.IndexSchema [backuprestore_shard2_replica_n4] Schema name=minimal
[junit4] 2> 40337 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.s.IndexSchema [backuprestore_shard1_replica_n1] Schema name=minimal
[junit4] 2> 40625 WARN (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40630 WARN (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieLongField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40655 WARN (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40659 WARN (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40670 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 40671 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica_n4' using configuration from collection backuprestore, trusted=true
[junit4] 2> 40671 WARN (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40679 WARN (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40680 WARN (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40782 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 40782 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica_p6' using configuration from collection backuprestore, trusted=true
[junit4] 2> 41017 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 41022 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 41022 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 41017 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica_t2' using configuration from collection backuprestore, trusted=true
[junit4] 2> 41028 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 41028 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica_t5' using configuration from collection backuprestore, trusted=true
[junit4] 2> 41022 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica_n1' using configuration from collection backuprestore, trusted=true
[junit4] 2> 41022 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica_p3' using configuration from collection backuprestore, trusted=true
[junit4] 2> 41228 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.core.backuprestore.shard1.replica_t2' (registry 'solr.core.backuprestore.shard1.replica_t2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41232 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.core.backuprestore.shard1.replica_p3' (registry 'solr.core.backuprestore.shard1.replica_p3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41232 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.core.backuprestore.shard1.replica_n1' (registry 'solr.core.backuprestore.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41232 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.core.backuprestore.shard2.replica_t5' (registry 'solr.core.backuprestore.shard2.replica_t5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41233 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.core.backuprestore.shard2.replica_p6' (registry 'solr.core.backuprestore.shard2.replica_p6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41253 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41263 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41268 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41268 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41270 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrCore [[backuprestore_shard2_replica_p6] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/backuprestore_shard2_replica_p6], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/./backuprestore_shard2_replica_p6/data/]
[junit4] 2> 41284 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SolrCore [[backuprestore_shard2_replica_t5] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/backuprestore_shard2_replica_t5], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/./backuprestore_shard2_replica_t5/data/]
[junit4] 2> 41299 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrCore [[backuprestore_shard1_replica_p3] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/backuprestore_shard1_replica_p3], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/./backuprestore_shard1_replica_p3/data/]
[junit4] 2> 41363 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrCore [[backuprestore_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/backuprestore_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/./backuprestore_shard1_replica_n1/data/]
[junit4] 2> 41367 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41367 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore [[backuprestore_shard1_replica_t2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/backuprestore_shard1_replica_t2], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/./backuprestore_shard1_replica_t2/data/]
[junit4] 2> 41376 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.core.backuprestore.shard2.replica_n4' (registry 'solr.core.backuprestore.shard2.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41376 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41376 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrCore [[backuprestore_shard2_replica_n4] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/backuprestore_shard2_replica_n4], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/./backuprestore_shard2_replica_n4/data/]
[junit4] 2> 52558 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 52558 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 52668 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 52711 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 52863 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 52863 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 52876 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 53333 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 53392 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 53392 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 53394 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 53394 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 53423 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 53424 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 53437 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 53437 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 53439 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 53439 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 53469 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 53470 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 53513 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@4ec697d8[backuprestore_shard2_replica_n4] main]
[junit4] 2> 53532 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.s.SolrIndexSearcher Opening [Searcher@314e8040[backuprestore_shard2_replica_t5] main]
[junit4] 2> 53532 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@bc28726[backuprestore_shard1_replica_n1] main]
[junit4] 2> 53533 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@3347e5b6[backuprestore_shard1_replica_t2] main]
[junit4] 2> 53559 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.SolrIndexSearcher Opening [Searcher@372436bd[backuprestore_shard2_replica_p6] main]
[junit4] 2> 53624 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.SolrIndexSearcher Opening [Searcher@76bf53d[backuprestore_shard1_replica_p3] main]
[junit4] 2> 53665 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53667 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53708 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53715 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53715 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53731 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53740 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53740 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53755 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53760 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53765 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53775 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53841 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 53878 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 53897 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 53897 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 53848 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 53898 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 57876 WARN (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker-SendThread(127.0.0.1:35952)) [ ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 6712ms for sessionid 0x15f7083c0440008
[junit4] 2> 57877 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f7083c0440008, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 58083 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582742906708426752
[junit4] 2> 58264 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 58266 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 58284 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 59122 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.ZkController backuprestore_shard1_replica_p3 starting background replication from leader
[junit4] 2> 59125 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582742907801042944
[junit4] 2> 59125 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.ZkController backuprestore_shard2_replica_p6 starting background replication from leader
[junit4] 2> 59126 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582742907802091520
[junit4] 2> 59132 WARN (zkCallback-20-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5077fabf name: ZooKeeperConnection Watcher:127.0.0.1:35952/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 59132 WARN (zkCallback-20-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 59216 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 59293 INFO (searcherExecutor-20-thread-1-processing-n:127.0.0.1:58560_solr x:backuprestore_shard1_replica_n1 s:shard1 c:backuprestore r:core_node7) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrCore [backuprestore_shard1_replica_n1] Registered new searcher Searcher@bc28726[backuprestore_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59294 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 59294 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 59295 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:58560/solr/backuprestore_shard1_replica_n1/
[junit4] 2> 59421 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=backuprestore_shard1_replica_n1 url=http://127.0.0.1:58560/solr START replicas=[http://127.0.0.1:41201/solr/backuprestore_shard1_replica_t2/] nUpdates=100
[junit4] 2> 59770 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=backuprestore_shard1_replica_n1 url=http://127.0.0.1:58560/solr DONE. We have no versions. sync failed.
[junit4] 2> 59773 INFO (searcherExecutor-18-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard2_replica_p6 s:shard2 c:backuprestore r:core_node12) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrCore [backuprestore_shard2_replica_p6] Registered new searcher Searcher@372436bd[backuprestore_shard2_replica_p6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59777 INFO (searcherExecutor-21-thread-1-processing-n:127.0.0.1:58560_solr x:backuprestore_shard2_replica_t5 s:shard2 c:backuprestore r:core_node11) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SolrCore [backuprestore_shard2_replica_t5] Registered new searcher Searcher@314e8040[backuprestore_shard2_replica_t5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59779 INFO (searcherExecutor-22-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard1_replica_t2 s:shard1 c:backuprestore r:core_node8) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore [backuprestore_shard1_replica_t2] Registered new searcher Searcher@3347e5b6[backuprestore_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59780 INFO (searcherExecutor-17-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard2_replica_n4 s:shard2 c:backuprestore r:core_node10) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrCore [backuprestore_shard2_replica_n4] Registered new searcher Searcher@4ec697d8[backuprestore_shard2_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59782 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582742908489957376
[junit4] 2> 59231 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 59804 INFO (searcherExecutor-19-thread-1-processing-n:127.0.0.1:58560_solr x:backuprestore_shard1_replica_p3 s:shard1 c:backuprestore r:core_node9) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrCore [backuprestore_shard1_replica_p3] Registered new searcher Searcher@76bf53d[backuprestore_shard1_replica_p3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59845 INFO (qtp1468422210-28) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.S.Request [backuprestore_shard1_replica_t2] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=392
[junit4] 2> 59851 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=9999ms
[junit4] 2> 59882 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 59882 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 59883 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 59883 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 60015 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60016 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60417 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 60417 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 60417 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:58560/solr/backuprestore_shard2_replica_t5/
[junit4] 2> 60419 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.PeerSync PeerSync: core=backuprestore_shard2_replica_t5 url=http://127.0.0.1:58560/solr START replicas=[http://127.0.0.1:41201/solr/backuprestore_shard2_replica_n4/] nUpdates=100
[junit4] 2> 60531 INFO (qtp1468422210-31) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.S.Request [backuprestore_shard2_replica_n4] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 60531 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 60532 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 60532 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 60616 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:58560/solr/backuprestore_shard1_replica_n1/ shard1
[junit4] 2> 60717 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60717 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60731 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.PeerSync PeerSync: core=backuprestore_shard2_replica_t5 url=http://127.0.0.1:58560/solr DONE. We have no versions. sync failed.
[junit4] 2> 60731 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 60731 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 60732 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 60732 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 60736 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ZkController backuprestore_shard2_replica_t5 stopping background replication from leader
[junit4] 2> 60756 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:58560/solr/backuprestore_shard2_replica_t5/ shard2
[junit4] 2> 60757 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node7&name=backuprestore_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&wt=javabin} status=0 QTime=25431
[junit4] 2> 60759 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Replica core_node7 is leader but it's state is down, skipping replication
[junit4] 2> 60860 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60861 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60863 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 60880 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node11&name=backuprestore_shard2_replica_t5&action=CREATE&numShards=2&shard=shard2&wt=javabin} status=0 QTime=25599
[junit4] 2> 60882 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Replica core_node11 is leader but it's state is down, skipping replication
[junit4] 2> 60903 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node10&name=backuprestore_shard2_replica_n4&action=CREATE&numShards=2&shard=shard2&wt=javabin} status=0 QTime=25619
[junit4] 2> 60905 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node9&name=backuprestore_shard1_replica_p3&action=CREATE&numShards=2&shard=shard1&wt=javabin} status=0 QTime=25580
[junit4] 2> 60940 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node12&name=backuprestore_shard2_replica_p6&action=CREATE&numShards=2&shard=shard2&wt=javabin} status=0 QTime=25524
[junit4] 2> 61052 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 61053 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 61432 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.ZkController backuprestore_shard1_replica_t2 starting background replication from leader
[junit4] 2> 61432 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 61449 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 61449 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 61452 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node8&name=backuprestore_shard1_replica_t2&action=CREATE&numShards=2&shard=shard1&wt=javabin} status=0 QTime=26123
[junit4] 2> 61461 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:58560/solr/backuprestore_shard1_replica_n1/
[junit4] 2> 61615 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 61616 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 61628 INFO (qtp1422851308-36) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.S.Request [backuprestore_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=150
[junit4] 2> 61636 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 61648 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 61648 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 61648 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 61649 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 61676 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@7af42f8f[backuprestore_shard1_replica_t2] main]
[junit4] 2> 61705 INFO (searcherExecutor-22-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard1_replica_t2 s:shard1 c:backuprestore r:core_node8) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore [backuprestore_shard1_replica_t2] Registered new searcher Searcher@7af42f8f[backuprestore_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 61965 INFO (OverseerCollectionConfigSetProcessor-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 62660 INFO (qtp1422851308-35) [n:127.0.0.1:58560_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 62661 INFO (qtp1422851308-35) [n:127.0.0.1:58560_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&router.name=implicit&version=2&pullReplicas=1&shards=shard1,shard2&property.customKey=customValue&maxShardsPerNode=3&router.field=shard_s&autoAddReplicas=true&name=backuprestore&nrtReplicas=1&action=CREATE&tlogReplicas=1&wt=javabin} status=0 QTime=29127
[junit4] 2> 62883 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication
[junit4] 2> 62884 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:58560/solr/backuprestore_shard1_replica_n1/
[junit4] 2> 62885 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication
[junit4] 2> 62885 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:58560/solr/backuprestore_shard2_replica_t5/
[junit4] 2> 62886 INFO (qtp1422851308-39) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.S.Request [backuprestore_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 62892 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 62892 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 62892 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 62892 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 62892 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 62896 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 62896 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@2c8c1df2 commitCommandVersion:0
[junit4] 2> 62936 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.S.Request [backuprestore_shard2_replica_t5] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=2
[junit4] 2> 62937 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 62937 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 62937 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 62937 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 62937 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 62938 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 62938 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@70ddacb8 commitCommandVersion:0
[junit4] 2> 62952 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.SolrIndexSearcher Opening [Searcher@582d8e1b[backuprestore_shard1_replica_p3] main]
[junit4] 2> 62968 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.SolrIndexSearcher Opening [Searcher@1dbec7a1[backuprestore_shard2_replica_p6] main]
[junit4] 2> 62969 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 62970 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 63636 INFO (searcherExecutor-18-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard2_replica_p6 s:shard2 c:backuprestore r:core_node12) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrCore [backuprestore_shard2_replica_p6] Registered new searcher Searcher@1dbec7a1[backuprestore_shard2_replica_p6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 63699 INFO (searcherExecutor-19-thread-1-processing-n:127.0.0.1:58560_solr x:backuprestore_shard1_replica_p3 s:shard1 c:backuprestore r:core_node9) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrCore [backuprestore_shard1_replica_p3] Registered new searcher Searcher@582d8e1b[backuprestore_shard1_replica_p3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 64458 INFO (qtp1422851308-32) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.S.Request [backuprestore_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 64460 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 64460 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 64460 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 64460 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 64460 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 64480 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@5fd941fa[backuprestore_shard1_replica_t2] main]
[junit4] 2> 64483 INFO (searcherExecutor-22-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard1_replica_t2 s:shard1 c:backuprestore r:core_node8) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore [backuprestore_shard1_replica_t2] Registered new searcher Searcher@5fd941fa[backuprestore_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 65903 INFO (qtp1422851308-32) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.S.Request [backuprestore_shard2_replica_t5] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Slave's generation: 2
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Slave's version: 1509421264457
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@70ddacb8 commitCommandVersion:0
[junit4] 2> 65928 INFO (qtp1422851308-39) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.S.Request [backuprestore_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Slave's generation: 2
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Slave's version: 1509421264415
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@2c8c1df2 commitCommandVersion:0
[junit4] 2> 65948 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.SolrIndexSearcher Opening [Searcher@3aa70f9b[backuprestore_shard2_replica_p6] main]
[junit4] 2> 65948 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 65951 INFO (searcherExecutor-18-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard2_replica_p6 s:shard2 c:backuprestore r:core_node12) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrCore [backuprestore_shard2_replica_p6] Registered new searcher Searcher@3aa70f9b[backuprestore_shard2_replica_p6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 65953 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.SolrIndexSearcher Opening [Searcher@8d8fcd8[backuprestore_shard1_replica_p3] main]
[junit4] 2> 65953 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 65958 INFO (searcherExecutor-19-thread-1-processing-n:127.0.0.1:58560_solr x:backuprestore_shard1_replica_p3 s:shard1 c:backuprestore r:core_node9) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrCore [backuprestore_shard1_replica_p3] Registered new searcher Searcher@8d8fcd8[backuprestore_shard1_replica_p3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 67456 INFO (qtp1422851308-32) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.S.Request [backuprestore_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 67457 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 67457 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 67457 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 67457 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 67457 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 67471 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@4ed43433[backuprestore_shard1_replica_t2] main]
[junit4] 2> 67475 INFO (searcherExecutor-22-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard1_replica_t2 s:shard1 c:backuprestore r:core_node8) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore [backuprestore_shard1_replica_t2] Registered new searcher Searcher@4ed43433[backuprestore_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 67500 INFO (qtp1468422210-31) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [backuprestore_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:58560/solr/backuprestore_shard1_replica_n1/&wt=javabin&version=2}{add=[0 (1582742912532217856), 1 (1582742915467182080), 4 (1582742915491299328), 6 (1582742915497590784), 7 (1582742915514368000), 8 (1582742915646488576), 10 (1582742915649634304), 13 (1582742915684237312), 14 (1582742915689480192), 17 (1582742915
[...truncated too long message...]
ave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptor-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-admin-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-authn-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-authz-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/byte-buddy-1.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/commons-math3-3.6.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/mockito-core-2.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/objenesis-2.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/analysis/icu/lucene-analyzers-icu-7.2.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/backward-codecs/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/contrib/analysis-extras/lib/icu4j-59.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_144/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20171031_033935_3816929525940187101917.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20171031_033935_3814221955659416896517.suites -stdin
[junit4] ERROR: JVM J2 ended with an exception: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20171031_033935_3814842593785503032539.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20171031_033935_3814372273523413635019.syserr
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1519)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
[junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] at java.lang.Thread.run(Thread.java:748)
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:826: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:770: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build.xml:262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/common-build.xml:556: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1494: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1021: At least one slave process threw an exception, first: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J0-20171031_033935_3321840776022627920860.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J0-20171031_033935_3326211161918442131645.syserr
Total time: 235 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-Tests-7.x #197 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 198 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/198/
No tests ran.
Build Log:
[...truncated 7005 lines...]
ERROR: command execution failed.
ERROR: Step ‘Archive the artifacts’ failed: no workspace for Lucene-Solr-Tests-7.x #198
ERROR: Step ‘Publish JUnit test result report’ failed: no workspace for Lucene-Solr-Tests-7.x #198
ERROR: lucene is offline; cannot locate JDK 1.8 (latest)
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
ERROR: lucene is offline; cannot locate JDK 1.8 (latest)
ERROR: lucene is offline; cannot locate JDK 1.8 (latest)
[JENKINS] Lucene-Solr-Tests-7.x - Build # 197 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/197/
10 tests failed.
FAILED: org.apache.solr.cloud.ShardRoutingTest.test
Error Message:
Error from server at http://127.0.0.1:34143: At least one of the node(s) specified are not currently active, no action taken.
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:34143: At least one of the node(s) specified are not currently active, no action taken.
at __randomizedtesting.SeedInfo.seed([858C689B840B3A43:DD857412AF757BB]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1096)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:875)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:808)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:418)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ShardRoutingTest
Error Message:
198 threads leaked from SUITE scope at org.apache.solr.cloud.ShardRoutingTest: 1) Thread[id=4679, name=qtp383150318-4679, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=4599, name=Scheduler-351568579, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=4534, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 4) Thread[id=4607, name=qtp331754181-4607-acceptor-0@17c44e2f-ServerConnector@67347bd3{HTTP/1.1,[http/1.1]}{127.0.0.1:59846}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=4588, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=4730, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 7) Thread[id=4675, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=4697, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-EventThread, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 9) Thread[id=4504, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 10) Thread[id=4493, name=qtp2059076216-4493, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=4584, name=org.eclipse.jetty.server.session.HashSessionManager@255a5ef9Timer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=4764, name=zkCallback-809-thread-4, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=4706, name=updateExecutor-832-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=4523, name=qtp1505717836-4523, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=4748, name=zkCallback-797-thread-4, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=4543, name=searcherExecutor-1935-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=4705, name=searcherExecutor-1990-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=4530, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=4650, name=org.eclipse.jetty.server.session.HashSessionManager@1ff96782Timer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=4754, name=zkCallback-842-thread-2, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=4671, name=searcherExecutor-1979-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=4711, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=4610, name=qtp331754181-4610, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=4756, name=zkCallback-842-thread-4, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=4618, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=4554, name=org.eclipse.jetty.server.session.HashSessionManager@e00fddcTimer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=4647, name=qtp1351042721-4647, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 28) Thread[id=4769, name=zkCallback-821-thread-5, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=4721, name=org.eclipse.jetty.server.session.HashSessionManager@6510dcd6Timer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=4613, name=org.eclipse.jetty.server.session.HashSessionManager@635d8de3Timer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 31) Thread[id=4500, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=4681, name=qtp383150318-4681-acceptor-0@603f0db1-ServerConnector@79281ec4{HTTP/1.1,[http/1.1]}{127.0.0.1:50183}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=4762, name=zkCallback-809-thread-2, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 34) Thread[id=4668, name=watches-829-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=4725, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=4578, name=qtp1094907735-4578-acceptor-0@1d2723af-ServerConnector@32599859{HTTP/1.1,[http/1.1]}{127.0.0.1:37045}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 37) Thread[id=4563, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 38) Thread[id=4595, name=zkCallback-815-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 39) Thread[id=4682, name=qtp383150318-4682, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 40) Thread[id=4715, name=qtp1837148975-4715, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 41) Thread[id=4698, name=zkCallback-835-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 42) Thread[id=4719, name=qtp1837148975-4719-acceptor-0@38f54143-ServerConnector@2b2cf0f1{HTTP/1.1,[http/1.1]}{127.0.0.1:40817}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 43) Thread[id=4524, name=qtp1505717836-4524, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 44) Thread[id=4511, name=Scheduler-1049167543, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 45) Thread[id=4535, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-EventThread, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 46) Thread[id=4536, name=zkCallback-803-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 47) Thread[id=4773, name=zkCallback-828-thread-5, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 48) Thread[id=4684, name=qtp383150318-4684, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 49) Thread[id=4577, name=qtp1094907735-4577, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 50) Thread[id=4643, name=qtp1351042721-4643, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 51) Thread[id=4488, name=qtp2059076216-4488, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 52) Thread[id=4759, name=zkCallback-815-thread-3, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 53) Thread[id=4519, name=qtp1505717836-4519-acceptor-0@75539a1e-ServerConnector@321d6f5c{HTTP/1.1,[http/1.1]}{127.0.0.1:38072}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 54) Thread[id=4624, name=zkCallback-821-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 55) Thread[id=4776, name=zkCallback-835-thread-4, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 56) Thread[id=4752, name=zkCallback-803-thread-4, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 57) Thread[id=4770, name=zkCallback-828-thread-2, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 58) Thread[id=4579, name=qtp1094907735-4579, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 59) Thread[id=4645, name=qtp1351042721-4645, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 60) Thread[id=4529, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 61) Thread[id=4546, name=qtp21053055-4546, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 62) Thread[id=4696, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 63) Thread[id=4648, name=qtp1351042721-4648, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 64) Thread[id=4766, name=zkCallback-821-thread-2, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 65) Thread[id=4665, name=coreContainerWorkExecutor-1974-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 66) Thread[id=4573, name=searcherExecutor-1946-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 67) Thread[id=4514, name=searcherExecutor-1924-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 68) Thread[id=4718, name=qtp1837148975-4718, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 69) Thread[id=4751, name=zkCallback-803-thread-3, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 70) Thread[id=4608, name=qtp331754181-4608, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 71) Thread[id=4644, name=qtp1351042721-4644-acceptor-0@7ce398e3-ServerConnector@3bd4dc11{HTTP/1.1,[http/1.1]}{127.0.0.1:40270}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 72) Thread[id=4699, name=Thread-1013, state=WAITING, group=TGRP-ShardRoutingTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 73) Thread[id=4553, name=qtp21053055-4553, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 74) Thread[id=4717, name=qtp1837148975-4717, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 75) Thread[id=4749, name=zkCallback-797-thread-5, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 76) Thread[id=4753, name=zkCallback-803-thread-5, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 77) Thread[id=4537, name=Thread-968, state=WAITING, group=TGRP-ShardRoutingTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 78) Thread[id=4758, name=zkCallback-815-thread-2, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 79) Thread[id=4767, name=zkCallback-821-thread-3, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 80) Thread[id=4558, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 81) Thread[id=4771, name=zkCallback-828-thread-3, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 82) Thread[id=4583, name=qtp1094907735-4583, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 83) Thread[id=4507, name=Thread-959, state=WAITING, group=TGRP-ShardRoutingTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 84) Thread[id=4489, name=qtp2059076216-4489-acceptor-0@283458db-ServerConnector@7a134e2f{HTTP/1.1,[http/1.1]}{127.0.0.1:36211}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 85) Thread[id=4633, name=searcherExecutor-1968-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 86) Thread[id=4700, name=coreContainerWorkExecutor-1985-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 87) Thread[id=4622, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 88) Thread[id=4506, name=zkCallback-797-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 89) Thread[id=4520, name=qtp1505717836-4520, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 90) Thread[id=4525, name=org.eclipse.jetty.server.session.HashSessionManager@38d81390Timer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 91) Thread[id=4716, name=qtp1837148975-4716, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 92) Thread[id=4550, name=qtp21053055-4550, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 93) Thread[id=4740, name=OverseerCollectionConfigSetProcessor-98916296398602249-127.0.0.1:36211_-n_0000000001, state=TIMED_WAITING, group=Overseer collection creation process.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 94) Thread[id=4663, name=zkCallback-828-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 95) Thread[id=4691, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 96) Thread[id=4547, name=qtp21053055-4547, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 97) Thread[id=4636, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 98) Thread[id=4487, name=qtp2059076216-4487, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 99) Thread[id=4491, name=qtp2059076216-4491, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 100) Thread[id=4739, name=OverseerStateUpdate-98916296398602249-127.0.0.1:36211_-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:435) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:202) at java.lang.Thread.run(Thread.java:748) 101) Thread[id=4551, name=qtp21053055-4551, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 102) Thread[id=4726, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 103) Thread[id=4657, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 104) Thread[id=4649, name=qtp1351042721-4649, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 105) Thread[id=4714, name=qtp1837148975-4714, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 106) Thread[id=4692, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 107) Thread[id=4755, name=zkCallback-842-thread-3, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 108) Thread[id=4494, name=qtp2059076216-4494, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 109) Thread[id=4617, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 110) Thread[id=4778, name=ScheduledTrigger-2004-thread-3, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 111) Thread[id=4559, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 112) Thread[id=4549, name=qtp21053055-4549, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 113) Thread[id=4548, name=qtp21053055-4548-acceptor-0@78d4bb24-ServerConnector@3109465d{HTTP/1.1,[http/1.1]}{127.0.0.1:34143}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 114) Thread[id=4545, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 115) Thread[id=4512, name=watches-798-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 116) Thread[id=4566, name=Thread-977, state=WAITING, group=TGRP-ShardRoutingTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 117) Thread[id=4777, name=zkCallback-835-thread-5, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurren
[...truncated too long message...]
rent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 18) Thread[id=6926, name=TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45890), state=TIMED_WAITING, group=TGRP-ShardSplitTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > 19) Thread[id=6900, name=Thread-1417, state=WAITING, group=TGRP-ShardSplitTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678)
[junit4] > 20) Thread[id=6881, name=qtp1732939943-6881, state=RUNNABLE, group=TGRP-ShardSplitTest]
[junit4] > at sun.nio.ch.IOUtil.drain(Native Method)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 21) Thread[id=6871, name=Thread-1408, state=WAITING, group=TGRP-ShardSplitTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678)
[junit4] > 22) Thread[id=6943, name=qtp199264780-6943, state=RUNNABLE, group=TGRP-ShardSplitTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
[junit4] > at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
[junit4] > at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
[junit4] > at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 23) Thread[id=6935, name=searcherExecutor-3171-thread-1, state=WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 24) Thread[id=6874, name=Scheduler-821479584, state=WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 25) Thread[id=6884, name=qtp1732939943-6884, state=RUNNABLE, group=TGRP-ShardSplitTest]
[junit4] > at sun.nio.ch.IOUtil.drain(Native Method)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 26) Thread[id=6915, name=qtp551973238-6915, state=RUNNABLE, group=TGRP-ShardSplitTest]
[junit4] > at sun.nio.ch.IOUtil.drain(Native Method)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([858C689B840B3A43]:0)
[junit4] Completed [688/745 (5!)] on J0 in 578.91s, 5 tests, 1 failure, 3 errors <<< FAILURES!
[...truncated 47426 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 196 - Still unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/196/
1 tests failed.
FAILED: org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader
Error Message:
Could not load collection from ZK: forceleader_test_collection
Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: forceleader_test_collection
at __randomizedtesting.SeedInfo.seed([8BC74AD0A95B1D43:6D507E1090D9E422]:0)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
at org.apache.solr.common.cloud.ZkStateReader.getLeader(ZkStateReader.java:787)
at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:809)
at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:547)
at org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader(ForceLeaderTest.java:143)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/forceleader_test_collection/state.json
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
... 47 more
Build Log:
[...truncated 13453 lines...]
[junit4] Suite: org.apache.solr.cloud.ForceLeaderTest
[junit4] 2> 5056756 INFO (SUITE-ForceLeaderTest-seed#[8BC74AD0A95B1D43]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/init-core-data-001
[junit4] 2> 5056757 WARN (SUITE-ForceLeaderTest-seed#[8BC74AD0A95B1D43]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=14 numCloses=14
[junit4] 2> 5056757 INFO (SUITE-ForceLeaderTest-seed#[8BC74AD0A95B1D43]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 5056791 INFO (SUITE-ForceLeaderTest-seed#[8BC74AD0A95B1D43]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 5056791 INFO (SUITE-ForceLeaderTest-seed#[8BC74AD0A95B1D43]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /_/e
[junit4] 2> 5056848 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 5056848 INFO (Thread-2793) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 5056848 INFO (Thread-2793) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 5056883 ERROR (Thread-2793) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 5056956 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkTestServer start zk server on port:39184
[junit4] 2> 5057031 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 5057032 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 5057045 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 5057046 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 5057047 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 5057047 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 5057048 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 5057048 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 5057049 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 5057083 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 5057084 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 5057103 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
[junit4] 2> 5058141 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 5058284 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@63404405{/_/e,null,AVAILABLE}
[junit4] 2> 5058284 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@705906fa{HTTP/1.1,[http/1.1]}{127.0.0.1:37146}
[junit4] 2> 5058284 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server Started @5075560ms
[junit4] 2> 5058285 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/tempDir-001/control/data, replicaType=NRT, hostContext=/_/e, hostPort=43207, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/control-001/cores}
[junit4] 2> 5058285 ERROR (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5058285 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 5058285 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5058285 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5058285 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-28T10:57:06.126Z
[junit4] 2> 5058305 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 5058305 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/control-001/solr.xml
[junit4] 2> 5058378 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 5058378 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 5058379 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5058518 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39184/solr
[junit4] 2> 5059052 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5059065 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43207__%2Fe
[junit4] 2> 5059066 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.Overseer Overseer (id=98906159494201348-127.0.0.1:43207__%2Fe-n_0000000000) starting
[junit4] 2> 5059140 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43207__%2Fe
[junit4] 2> 5059175 INFO (zkCallback-1780-thread-1-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5059726 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5059872 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5059873 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5059907 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/control-001/cores
[junit4] 2> 5060110 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5060111 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39184/solr ready
[junit4] 2> 5060111 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=45701,localport=43207], receiveBufferSize:531000
[junit4] 2> 5060111 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37146,localport=58176], receiveBufferSize=530904
[junit4] 2> 5060165 INFO (qtp1430705358-12240) [n:127.0.0.1:43207__%2Fe ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:43207__%252Fe&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5060210 INFO (OverseerThreadFactory-5958-thread-1-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 5060353 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=45704,localport=43207], receiveBufferSize:531000
[junit4] 2> 5060364 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37146,localport=58180], receiveBufferSize=530904
[junit4] 2> 5060373 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 5060373 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 5060697 INFO (zkCallback-1780-thread-1-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 5061673 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 5061766 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 5062892 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 5063355 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 5063356 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5063356 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5063356 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 5063407 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39538988765470406]
[junit4] 2> 5063429 WARN (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 5064014 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5064014 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5064015 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5064015 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5064016 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1723384108, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 5064065 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4817dc57[control_collection_shard1_replica_n1] main]
[junit4] 2> 5064066 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 5064067 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 5064067 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 5064068 INFO (searcherExecutor-5961-thread-1-processing-n:127.0.0.1:43207__%2Fe x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@4817dc57[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5064069 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582498559463260160
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:43207/_/e/control_collection_shard1_replica_n1/
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:43207/_/e/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 5064140 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:43207/_/e/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 5064224 INFO (zkCallback-1780-thread-2-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 5064248 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 5064249 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3876
[junit4] 2> 5064306 INFO (qtp1430705358-12240) [n:127.0.0.1:43207__%2Fe ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 5064448 INFO (zkCallback-1780-thread-1-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 5065306 INFO (qtp1430705358-12240) [n:127.0.0.1:43207__%2Fe ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:43207__%252Fe&wt=javabin&version=2} status=0 QTime=5141
[junit4] 2> 5065490 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5065491 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39184/solr ready
[junit4] 2> 5065491 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 5065491 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=45744,localport=43207], receiveBufferSize:531000
[junit4] 2> 5065492 INFO (qtp1430705358-12241) [n:127.0.0.1:43207__%2Fe ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5065513 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37146,localport=58220], receiveBufferSize=530904
[junit4] 2> 5065526 INFO (OverseerCollectionConfigSetProcessor-98906159494201348-127.0.0.1:43207__%2Fe-n_0000000000) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5065526 INFO (OverseerThreadFactory-5958-thread-2-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 5065527 WARN (OverseerThreadFactory-5958-thread-2-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 5065783 INFO (qtp1430705358-12241) [n:127.0.0.1:43207__%2Fe ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 5065783 INFO (qtp1430705358-12241) [n:127.0.0.1:43207__%2Fe ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=291
[junit4] 2> 5067243 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001 of type TLOG
[junit4] 2> 5067243 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 5067396 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@25b7b6f9{/_/e,null,AVAILABLE}
[junit4] 2> 5067396 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@109cddf1{HTTP/1.1,[http/1.1]}{127.0.0.1:36794}
[junit4] 2> 5067396 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server Started @5084672ms
[junit4] 2> 5067396 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/_/e, hostPort=42162, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001/cores}
[junit4] 2> 5067396 ERROR (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5067396 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 5067397 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5067397 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5067397 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-28T10:57:15.238Z
[junit4] 2> 5067543 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 5067544 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001/solr.xml
[junit4] 2> 5067544 INFO (OverseerCollectionConfigSetProcessor-98906159494201348-127.0.0.1:43207__%2Fe-n_0000000000) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5067599 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 5067599 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 5067615 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5067637 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39184/solr
[junit4] 2> 5067765 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5067766 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5067800 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42162__%2Fe
[junit4] 2> 5067801 INFO (zkCallback-1780-thread-2-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5067920 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5067930 INFO (zkCallback-1788-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5069101 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5069192 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5069192 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5069226 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001/cores
[junit4] 2> 5069393 INFO (SocketProxy-Acceptor-42162) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=41154,localport=42162], receiveBufferSize:531000
[junit4] 2> 5069461 INFO (SocketProxy-Acceptor-42162) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=36794,localport=34552], receiveBufferSize=530904
[junit4] 2> 5069464 INFO (qtp1028606452-12294) [n:127.0.0.1:42162__%2Fe ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:42162__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5069520 INFO (OverseerThreadFactory-5958-thread-3-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:42162__%2Fe for creating new replica
[junit4] 2> 5069539 INFO (SocketProxy-Acceptor-42162) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=41158,localport=42162], receiveBufferSize:531000
[junit4] 2> 5069588 INFO (SocketProxy-Acceptor-42162) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=36794,localport=34556], receiveBufferSize=530904
[junit4] 2> 5069588 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 5069588 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 5069809 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 5069919 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema [collection1_shard2_replica_t41] Schema name=test
[junit4] 2> 5070974 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 5071066 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t41' using configuration from collection collection1, trusted=true
[junit4] 2> 5071067 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t41' (registry 'solr.core.collection1.shard2.replica_t41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5071067 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5071067 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [[collection1_shard2_replica_t41] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001/cores/collection1_shard2_replica_t41], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001/cores/collection1_shard2_replica_t41/data/]
[junit4] 2> 5071102 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39538988765470406]
[junit4] 2> 5071175 WARN (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 5071539 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5071539 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5071541 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5071541 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5071561 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1723384108, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 5071561 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.SolrIndexSearcher Opening [Searcher@74c1f867[collection1_shard2_replica_t41] main]
[junit4] 2> 5071576 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 5071577 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 5071577 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 5071578 INFO (searcherExecutor-5972-thread-1-processing-n:127.0.0.1:42162__%2Fe x:collection1_shard2_replica_t41 s:shard2 c:collection1) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [collection1_shard2_replica_t41] Registered new searcher Searcher@74c1f867[collection1_shard2_replica_t41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5071578 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582498567337017344
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:42162/_/e/collection1_shard2_replica_t41/
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy http://127.0.0.1:42162/_/e/collection1_shard2_replica_t41/ has no replicas
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 5071665 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController collection1_shard2_replica_t41 stopping background replication from leader
[junit4] 2> 5071685 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5071685 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:42162/_/e/collection1_shard2_replica_t41/ shard2
[junit4] 2> 5071686 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 5071686 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2098
[junit4] 2> 5071688 INFO (qtp1028606452-12294) [n:127.0.0.1:42162__%2Fe ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:42162__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=2223
[junit4] 2> 5071958 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5073318 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001 of type TLOG
[junit4] 2> 5073319 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 5073427 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7c2fdca1{/_/e,null,AVAILABLE}
[junit4] 2> 5073427 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5543f653{HTTP/1.1,[http/1.1]}{127.0.0.1:33348}
[junit4] 2> 5073427 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server Started @5090703ms
[junit4] 2> 5073427 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/_/e, hostPort=41891, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001/cores}
[junit4] 2> 5073428 ERROR (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5073465 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 5073465 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5073465 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5073465 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-28T10:57:21.306Z
[junit4] 2> 5073502 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 5073503 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001/solr.xml
[junit4] 2> 5073516 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f62a0c6c0000a, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 5073518 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 5073518 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 5073520 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5073553 INFO (OverseerCollectionConfigSetProcessor-98906159494201348-127.0.0.1:43207__%2Fe-n_0000000000) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5073569 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39184/solr
[junit4] 2> 5073755 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 5073772 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5073774 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41891__%2Fe
[junit4] 2> 5073774 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5073886 INFO (zkCallback-1788-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5073959 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5073882 INFO (zkCallback-1780-thread-2-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5075013 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5075124 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5075125 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5075126 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001/cores
[junit4] 2> 5075526 INFO (SocketProxy-Acceptor-41891) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=38412,localport=41891], receiveBufferSize:531000
[junit4] 2> 5075582 INFO (SocketProxy-Acceptor-41891) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=33348,localport=47020], receiveBufferSize=530904
[junit4] 2> 5075603 INFO (qtp994154853-12329) [n:127.0.0.1:41891__%2Fe ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41891__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5075638 INFO (OverseerThreadFactory-5958-thread-4-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:41891__%2Fe for creating new replica
[junit4] 2> 5075692 INFO (SocketProxy-Acceptor-41891) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=38416,localport=41891], receiveBufferSize:531000
[junit4] 2> 5075693 INFO (SocketProxy-Acceptor-41891) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=33348,localport=47024], receiveBufferSize=530904
[junit4] 2> 5075729 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 5075730 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 5075746 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 5076118 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 5076209 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema [collection1_shard1_replica_t43] Schema name=test
[junit4] 2> 5077525 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 5077675 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t43' using configuration from collection collection1, trusted=true
[junit4] 2> 5077676 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t43' (registry 'solr.core.collection1.shard1.replica_t43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5077676 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5077676 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [[collection1_shard1_replica_t43] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001/cores/collection1_shard1_replica_t43], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001/cores/collection1_shard1_replica_t43/data/]
[junit4] 2> 5077691 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39538988765470406]
[junit4] 2> 5077729 WARN (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 5078419 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5078419 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5078420 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5078420 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5078421 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1723384108, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 5078422 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.SolrIndexSearcher Opening [Searcher@54678c50[collection1_shard1_replica_t43] main]
[junit4] 2> 5078439 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 5078439 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 5078440 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 5078440 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582498574532345856
[junit4] 2> 5078442 INFO (searcherExecutor-5983-thread-1-processing-n:127.0.0.1:41891__%2Fe x:collection1_shard1_replica_t43 s:shard1 c:collection1) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [collection1_shard1_replica_t43] Registered new searcher Searcher@54678c50[collection1_shard1_replica_t43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:41891/_/e/collection1_shard1_replica_t43/
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy http://127.0.0.1:41891/_/e/collection1_shard1_replica_t43/ has no replicas
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController collection1_shard1_replica_t43 stopping background replication from leader
[junit4] 2> 5078471 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:41891/_/e/collection1_shard1_replica_t43/ shard1
[junit4] 2> 5078493 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 5078499 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 5078528 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 5078530 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2801
[junit4] 2> 5078593 INFO (qtp994154853-12329) [n:127.0.0.1:41891__%2Fe ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:41891__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=2990
[junit4] 2> 5078639 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 5078646 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 5079651 INFO (OverseerCollectionConfigSetProcessor-98906159494201348-127.0.0.1:43207__%2Fe-n_0000000000) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5080691 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001 of type TLOG
[junit4] 2> 5080708 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 5080952 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4a46eab2{/_/e,null,AVAILABLE}
[junit4] 2> 5080953 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6b991fb7{HTTP/1.1,[http/1.1]}{127.0.0.1:39542}
[junit4] 2> 5080953 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server Started @5098228ms
[junit4] 2> 5080953 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/tempDir-001/jetty3, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/_/e, hostPort=41221, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001/cores}
[junit4] 2> 5080953 ERROR (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5080962 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 5080962 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5080962 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5080963 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-28T10:57:28.804Z
[junit4] 2> 5081054 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 5081055 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001/solr.xml
[junit4] 2> 5081074 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 5081074 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 5081145 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5081165 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39184/solr
[junit4] 2> 5081367 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 5081384 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5081385 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41221__%2Fe
[junit4] 2> 5081400 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5081496 INFO (zkCallback-1805-thread-1-processing-n:127.0.0.1:41221__%2Fe) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5081461 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5081461 INFO (zkCallback-1780-thread-1-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5081461 INFO (zkCallback-1788-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5081951 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5082100 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5082133 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5082135 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001/cores
[junit4] 2> 5082537 INFO (SocketProxy-Acceptor-41221) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=53178,localport=41221], receiveBufferSize:531000
[junit4] 2> 5082556 INFO (SocketProxy-Acceptor-41221) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=39542,localport=40134], receiveBufferSize=530904
[junit4] 2> 5082596 INFO (qtp1223044078-12363) [n:127.0.0.1:41221__%2Fe ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41221__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5082704 INFO (OverseerThreadFactory-5958-thread-5-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:41221__%2Fe for creating new replica
[junit4] 2> 5082705 INFO (SocketProxy-Acceptor-41221) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=53182,localport=41221], receiveBufferSize:531000
[junit4] 2> 5082772 INFO (SocketProxy-Acceptor-41221) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=39542,localport=40138], receiveBufferSize=530904
[junit4] 2> 5082772 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 5082773 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 5082794 INFO (zkCallback-1805-thread-1-processing-n:127.0.0.1:41221__%2Fe) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5082794 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5082794 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5083918 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 5084193 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.s.IndexSchema [collection1_shard2_replica_t45] Schema name=test
[junit4] 2> 5085348 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 5085438 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t45' using configuration from collection collection1, trusted=true
[junit4] 2> 5085439 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t45' (registry 'solr.core.collection1.shard2.replica_t45') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5085439 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5085439 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.SolrCore [[collection1_shard2_replica_t45] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001/cores/collection1_shard2_replica_t45], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001/cores/collection1_shard2_replica_t45/data/]
[junit4] 2> 5085494 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39538988765470406]
[junit4] 2> 5085511 WARN (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 5086095 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5086095 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5086096 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5086096 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5086097 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1723384108, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 5086098 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.s.SolrIndexSearcher Opening [Searcher@234ce6ec[collection1_shard2_replica_t45] main]
[junit4] 2> 5086132 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 5086132 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 5086133 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 5086134 INFO (searcherExecutor-5994-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.SolrCore [collection1_shard2_replica_t45] Registered new searcher Searcher@234ce6ec[collection1_shard2_replica_t45] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5086134 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582498582600089600
[junit4] 2> 5086186 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.ZkController Core needs to recover:collection1_shard2_replica_t45
[junit4] 2> 5086187 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=3414
[junit4] 2> 5086188 INFO (updateExecutor-1802-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 5086201 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 5086278 INFO (qtp1223044078-12363) [n:127.0.0.1:41221__%2Fe ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:41221__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=3682
[junit4] 2> 5086331 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 5086331 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.ZkController collection1_shard2_replica_t45 stopping background replication from leader
[junit4] 2> 5086331 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard2_replica_t45]
[junit4] 2> 5086331 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 5086331 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard2_replica_t45] as recovering, leader is [http://127.0.0.1:42162/_/e/collection1_shard2_replica_t41/] and I am [http://127.0.0.1:41221/_/e/collection1_shard2_replica_t45/]
[junit4] 2> 5086332 INFO (zkCallback-1805-thread-1-processing-n:127.0.0.1:41221__%2Fe) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5086335 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5086335 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5086344 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testLastPublishedStateIsActive
[junit4] 2> 5086345 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:4216
[...truncated too long message...]
> 5212092 INFO (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40794 40794
[junit4] 2> 5212367 INFO (Thread-2834) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40794 40794
[junit4] 2> 5212387 WARN (Thread-2834) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/aliases.json
[junit4] 2> 4 /solr/security.json
[junit4] 2> 4 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/clusterprops.json
[junit4] 2> 7 /solr/clusterstate.json
[junit4] 2> 3 /solr/collections/forceleader_test_collection/state.json
[junit4] 2> 3 /solr/collections/collection1/state.json
[junit4] 2> 3 /solr/autoscaling.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/live_nodes
[junit4] 2> 7 /solr/collections
[junit4] 2> 3 /solr/overseer/collection-queue-work
[junit4] 2> 3 /solr/overseer/queue-work
[junit4] 2> 2 /solr/overseer/queue
[junit4] 2> 2 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2>
[junit4] 2> 5212387 WARN (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SocketProxy Closing 16 connections to: http://127.0.0.1:41867/_/e, target: http://127.0.0.1:34584/_/e
[junit4] 2> 5212400 WARN (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SocketProxy Closing 0 connections to: http://127.0.0.1:36363/_/e, target: http://127.0.0.1:43221/_/e
[junit4] 2> 5212400 WARN (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SocketProxy Closing 4 connections to: http://127.0.0.1:43097/_/e, target: http://127.0.0.1:40800/_/e
[junit4] 2> 5212400 WARN (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SocketProxy Closing 7 connections to: http://127.0.0.1:41330/_/e, target: http://127.0.0.1:43778/_/e
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ForceLeaderTest -Dtests.method=testReplicasInLIRNoLeader -Dtests.seed=8BC74AD0A95B1D43 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=tr-TR -Dtests.timezone=Africa/Ouagadougou -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 64.6s J1 | ForceLeaderTest.testReplicasInLIRNoLeader <<<
[junit4] > Throwable #1: org.apache.solr.common.SolrException: Could not load collection from ZK: forceleader_test_collection
[junit4] > at __randomizedtesting.SeedInfo.seed([8BC74AD0A95B1D43:6D507E1090D9E422]:0)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
[junit4] > at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.getLeader(ZkStateReader.java:787)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:809)
[junit4] > at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:547)
[junit4] > at org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader(ForceLeaderTest.java:143)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/forceleader_test_collection/state.json
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] > at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
[junit4] > ... 47 more
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001
[junit4] 2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=7901, maxDocsPerChunk=2, blockSize=5), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=7901, blockSize=5)), sim=RandomSimilarity(queryNorm=true): {}, locale=tr-TR, timezone=Africa/Ouagadougou
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=274698120,total=522715136
[junit4] 2> NOTE: All tests run in this JVM: [OverseerCollectionConfigSetProcessorTest, TestLazyCores, TestCloudNestedDocsSort, DocValuesMultiTest, DistributedTermsComponentTest, TestOverriddenPrefixQueryForCustomFieldType, TestMaxTokenLenTokenizer, TestExactStatsCache, OpenExchangeRatesOrgProviderTest, HdfsUnloadDistributedZkTest, SolrJmxReporterTest, TestComplexPhraseQParserPlugin, SolrGangliaReporterTest, DirectUpdateHandlerTest, TestZkAclsWithHadoopAuth, CdcrBootstrapTest, TestDefaultStatsCache, DistributedDebugComponentTest, TestSolr4Spatial, TestConfigSetsAPIZkFailure, DistributedFacetPivotLargeTest, TestCloudPivotFacet, TestRTGBase, TestDynamicFieldResource, ShardRoutingTest, CoreAdminRequestStatusTest, TestPseudoReturnFields, TestSolrQueryParser, TestSolrCoreSnapshots, SolrRequestParserTest, TestMacros, HighlighterConfigTest, MBeansHandlerTest, TestScoreJoinQPScore, CdcrRequestHandlerTest, SolrIndexSplitterTest, ConfigureRecoveryStrategyTest, TestManagedResource, TestBackupRepositoryFactory, NodeAddedTriggerTest, OverseerStatusTest, SimpleMLTQParserTest, V2StandaloneTest, TestMiniSolrCloudClusterSSL, TestDistributedStatsComponentCardinality, TestFieldTypeCollectionResource, TestAtomicUpdateErrorCases, PreAnalyzedFieldTest, TestDocumentBuilder, TestFastWriter, AssignTest, TestSegmentSorting, HighlighterMaxOffsetTest, BlobRepositoryCloudTest, TestSerializedLuceneMatchVersion, TestAnalyzedSuggestions, LukeRequestHandlerTest, TestPushWriter, TestManagedSchema, ShowFileRequestHandlerTest, TestSkipOverseerOperations, TestSearcherReuse, TestLegacyTerms, DeleteLastCustomShardedReplicaTest, DateRangeFieldTest, TestFieldCacheWithThreads, HttpPartitionTest, ShardRoutingCustomTest, TestInfoStreamLogging, TestCustomStream, TestRequestForwarding, TestBulkSchemaAPI, SolrIndexConfigTest, ConcurrentDeleteAndCreateCollectionTest, SampleTest, TestCollectionAPIs, HdfsSyncSliceTest, TestPointFields, TestCollationField, TestRangeQuery, TestPivotHelperCode, TestFilteredDocIdSet, SuggesterTSTTest, TestFaceting, TestHighFrequencyDictionaryFactory, TestBinaryResponseWriter, DistribDocExpirationUpdateProcessorTest, UninvertDocValuesMergePolicyTest, TestNumericRangeQuery64, TemplateUpdateProcessorTest, TestReRankQParserPlugin, SpatialRPTFieldTypeTest, TestSortByMinMaxFunction, TestStressRecovery, HLLSerializationTest, TestDocSet, CircularListTest, BlockJoinFacetRandomTest, TestStressReorder, TestDocTermOrds, TestWriterPerf, SpatialHeatmapFacetsTest, ConvertedLegacyTest, ConjunctionSolrSpellCheckerTest, SOLR749Test, TestExceedMaxTermLength, DistributedFacetPivotSmallTest, OpenCloseCoreStressTest, DeleteShardTest, TestQueryWrapperFilter, SharedFSAutoReplicaFailoverTest, HdfsBasicDistributedZk2Test, SSLMigrationTest, TestFieldCacheSort, TestRetrieveFieldsOptimizer, TestBM25SimilarityFactory, TestJettySolrRunner, TestUseDocValuesAsStored, DistributedQueryComponentOptimizationTest, AddReplicaTest, TestXmlQParser, SynonymTokenizerTest, TestDistributedGrouping, TestHashPartitioner, TestRealTimeGet, TestCoreContainer, TestGroupingSearch, PeerSyncTest, TestSort, BasicFunctionalityTest, DistributedQueryElevationComponentTest, CoreAdminHandlerTest, TestCSVLoader, NoCacheHeaderTest, SolrCoreCheckLockOnStartupTest, QueryEqualityTest, TestRemoteStreaming, TestSolrDeletionPolicy1, SolrInfoBeanTest, IndexBasedSpellCheckerTest, DisMaxRequestHandlerTest, DocumentAnalysisRequestHandlerTest, RequestHandlersTest, PathHierarchyTokenizerFactoryTest, FieldAnalysisRequestHandlerTest, LoggingHandlerTest, IndexSchemaTest, QueryParsingTest, BinaryUpdateRequestHandlerTest, TestPHPSerializedResponseWriter, AlternateDirectoryTest, TestSolrIndexConfig, CopyFieldTest, TestSolrDeletionPolicy2, NumericFieldsTest, OutputWriterTest, DirectSolrConnectionTest, TestPhraseSuggestions, TestPerFieldSimilarity, PrimUtilsTest, SpellingQueryConverterTest, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestUtils, SystemInfoHandlerTest, DistributedIntervalFacetingTest, TestSimpleTrackingShardHandler, BaseCdcrDistributedZkTest, CdcrReplicationDistributedZkTest, CdcrReplicationHandlerTest, ChaosMonkeySafeLeaderWithPullReplicasTest, DeleteInactiveReplicaTest, DistribJoinFromCollectionTest, ForceLeaderTest]
[junit4] Completed [531/745 (1!)] on J1 in 156.01s, 3 tests, 1 error, 1 skipped <<< FAILURES!
[...truncated 47937 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 195 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/195/
4 tests failed.
FAILED: org.apache.solr.cloud.PeerSyncReplicationTest.test
Error Message:
expected:<0> but was:<1>
Stack Trace:
java.lang.AssertionError: expected:<0> but was:<1>
at __randomizedtesting.SeedInfo.seed([F937FCE697463010:7163C33C39BA5DE8]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:202)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.schema.TestCloudSchemaless.test
Error Message:
Stack Trace:
java.lang.NullPointerException
at __randomizedtesting.SeedInfo.seed([F937FCE697463010:7163C33C39BA5DE8]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:925)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:808)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:106)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:71)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:85)
at org.apache.solr.schema.TestCloudSchemaless.test(TestCloudSchemaless.java:171)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.update.TestInPlaceUpdatesDistrib.test
Error Message:
inplace_updatable_float didn't match for replica at client: https://127.0.0.1:36896/_/collection1 expected:<102.0> but was:<101.0>
Stack Trace:
java.lang.AssertionError: inplace_updatable_float didn't match for replica at client: https://127.0.0.1:36896/_/collection1 expected:<102.0> but was:<101.0>
at __randomizedtesting.SeedInfo.seed([F937FCE697463010:7163C33C39BA5DE8]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.apache.solr.update.TestInPlaceUpdatesDistrib.assertReplicaValue(TestInPlaceUpdatesDistrib.java:980)
at org.apache.solr.update.TestInPlaceUpdatesDistrib.delayedReorderingFetchesMissingUpdateFromLeaderTest(TestInPlaceUpdatesDistrib.java:875)
at org.apache.solr.update.TestInPlaceUpdatesDistrib.test(TestInPlaceUpdatesDistrib.java:145)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory
Error Message:
expected:<8> but was:<10>
Stack Trace:
java.lang.AssertionError: expected:<8> but was:<10>
at __randomizedtesting.SeedInfo.seed([F937FCE697463010:94CB581B2D0ECF17]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:262)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12243 lines...]
[junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/init-core-data-001
[junit4] 2> 1984751 INFO (SUITE-PeerSyncReplicationTest-seed#[F937FCE697463010]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 1984752 INFO (SUITE-PeerSyncReplicationTest-seed#[F937FCE697463010]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 1984752 INFO (SUITE-PeerSyncReplicationTest-seed#[F937FCE697463010]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 1984752 INFO (SUITE-PeerSyncReplicationTest-seed#[F937FCE697463010]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 1984771 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1984778 INFO (Thread-2895) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1984778 INFO (Thread-2895) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1984778 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.ZkTestServer start zk server on port:33010
[junit4] 2> 1984884 ERROR (Thread-2895) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1985226 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1985227 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 1985228 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1985229 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1985242 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1985242 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1985243 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1985243 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1985244 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1985244 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1985245 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1985279 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f5d0a2bca0001, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1985444 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 1986250 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1986341 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@191ee19f{/,null,AVAILABLE}
[junit4] 2> 1986342 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@312b369f{SSL,[ssl, http/1.1]}{127.0.0.1:33900}
[junit4] 2> 1986342 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.Server Started @2002731ms
[junit4] 2> 1986342 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/tempDir-001/control/data, hostContext=/, hostPort=33900, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/control-001/cores}
[junit4] 2> 1986342 ERROR (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1986342 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 1986342 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1986342 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1986342 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-27T08:54:30.065Z
[junit4] 2> 1986446 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1986446 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/control-001/solr.xml
[junit4] 2> 1986496 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1986496 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1986497 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1986568 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33010/solr
[junit4] 2> 1986644 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f5d0a2bca0004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1988484 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33900_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1988485 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33900_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33900_
[junit4] 2> 1988485 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33900_ ] o.a.s.c.Overseer Overseer (id=98900015091154949-127.0.0.1:33900_-n_0000000000) starting
[junit4] 2> 1988537 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33900_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33900_
[junit4] 2> 1988598 INFO (OverseerStateUpdate-98900015091154949-127.0.0.1:33900_-n_0000000000) [n:127.0.0.1:33900_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1989050 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33900_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 1989103 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33900_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 1989103 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33900_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 1989104 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33900_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/control-001/cores
[junit4] 2> 1989436 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1989436 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33010/solr ready
[junit4] 2> 1989730 INFO (qtp882637231-6693) [n:127.0.0.1:33900_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:33900_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1989781 INFO (OverseerThreadFactory-2236-thread-1-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 1989981 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1989982 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1990091 INFO (zkCallback-784-thread-1-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1991287 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 1991413 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 1992262 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1992351 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 1992351 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 1992351 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1992351 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 1992386 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=32.7412109375, floorSegmentMB=0.9931640625, forceMergeDeletesPctAllowed=4.645882138600275, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4070763039360047
[junit4] 2> 1992462 WARN (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1992736 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1992736 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1992737 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1992737 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1992739 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=16.9990234375, floorSegmentMB=1.7177734375, forceMergeDeletesPctAllowed=29.80170751436371, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.25869791656211993
[junit4] 2> 1992739 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@63671baf[control_collection_shard1_replica_n1] main]
[junit4] 2> 1992740 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1992753 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1992753 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1992753 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582400249765298176
[junit4] 2> 1992774 INFO (searcherExecutor-2239-thread-1-processing-n:127.0.0.1:33900_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@63671baf[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1992775 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1992775 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1992775 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:33900/control_collection_shard1_replica_n1/
[junit4] 2> 1992775 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1992775 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:33900/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 1992775 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1992789 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:33900/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 1992790 INFO (zkCallback-784-thread-2-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1992846 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1992847 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2865
[junit4] 2> 1992849 INFO (qtp882637231-6693) [n:127.0.0.1:33900_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1992956 INFO (zkCallback-784-thread-2-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1993799 INFO (OverseerCollectionConfigSetProcessor-98900015091154949-127.0.0.1:33900_-n_0000000000) [n:127.0.0.1:33900_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1993856 INFO (qtp882637231-6693) [n:127.0.0.1:33900_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:33900_&wt=javabin&version=2} status=0 QTime=4126
[junit4] 2> 1993974 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1994003 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33010/solr ready
[junit4] 2> 1994003 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1994018 INFO (qtp882637231-6694) [n:127.0.0.1:33900_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1994092 INFO (OverseerThreadFactory-2236-thread-2-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 1994093 WARN (OverseerThreadFactory-2236-thread-2-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 1994397 INFO (qtp882637231-6694) [n:127.0.0.1:33900_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1994397 INFO (qtp882637231-6694) [n:127.0.0.1:33900_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=378
[junit4] 2> 1995416 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-1-001 of type NRT
[junit4] 2> 1995430 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1995490 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@43f243f0{/,null,AVAILABLE}
[junit4] 2> 1995491 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3f094b93{SSL,[ssl, http/1.1]}{127.0.0.1:33495}
[junit4] 2> 1995491 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.Server Started @2011880ms
[junit4] 2> 1995491 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=33495, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-1-001/cores}
[junit4] 2> 1995491 ERROR (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1995491 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 1995491 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1995491 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1995491 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-27T08:54:39.214Z
[junit4] 2> 1995566 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1995566 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-1-001/solr.xml
[junit4] 2> 1995602 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1995602 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1995603 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1995637 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33010/solr
[junit4] 2> 1995745 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33495_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1995746 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33495_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1995759 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33495_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33495_
[junit4] 2> 1995836 INFO (zkCallback-792-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1995963 INFO (zkCallback-797-thread-1-processing-n:127.0.0.1:33495_) [n:127.0.0.1:33495_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1995963 INFO (zkCallback-784-thread-1-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1996129 INFO (OverseerCollectionConfigSetProcessor-98900015091154949-127.0.0.1:33900_-n_0000000000) [n:127.0.0.1:33900_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1996276 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33495_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 1996346 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33495_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 1996346 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33495_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 1996348 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33495_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-1-001/cores
[junit4] 2> 1996622 INFO (qtp1900424130-6741) [n:127.0.0.1:33495_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:33495_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1996624 INFO (OverseerThreadFactory-2236-thread-3-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:33495_ for creating new replica
[junit4] 2> 1996734 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1996734 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1997868 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 1997994 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema [collection1_shard1_replica_n21] Schema name=test
[junit4] 2> 1999145 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1999275 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n21' using configuration from collection collection1, trusted=true
[junit4] 2> 1999275 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n21' (registry 'solr.core.collection1.shard1.replica_n21') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 1999275 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1999275 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [[collection1_shard1_replica_n21] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-1-001/cores/collection1_shard1_replica_n21], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-1-001/cores/collection1_shard1_replica_n21/data/]
[junit4] 2> 1999347 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=32.7412109375, floorSegmentMB=0.9931640625, forceMergeDeletesPctAllowed=4.645882138600275, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4070763039360047
[junit4] 2> 1999386 WARN (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1999844 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1999844 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1999845 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1999845 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1999846 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=16.9990234375, floorSegmentMB=1.7177734375, forceMergeDeletesPctAllowed=29.80170751436371, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.25869791656211993
[junit4] 2> 1999847 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.SolrIndexSearcher Opening [Searcher@52595c65[collection1_shard1_replica_n21] main]
[junit4] 2> 1999848 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1999862 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1999863 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1999897 INFO (searcherExecutor-2250-thread-1-processing-n:127.0.0.1:33495_ x:collection1_shard1_replica_n21 s:shard1 c:collection1) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [collection1_shard1_replica_n21] Registered new searcher Searcher@52595c65[collection1_shard1_replica_n21] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1999897 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582400257256325120
[junit4] 2> 1999900 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1999900 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1999900 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:33495/collection1_shard1_replica_n21/
[junit4] 2> 1999900 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1999900 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy https://127.0.0.1:33495/collection1_shard1_replica_n21/ has no replicas
[junit4] 2> 1999900 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1999918 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:33495/collection1_shard1_replica_n21/ shard1
[junit4] 2> 1999970 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1999971 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3237
[junit4] 2> 1999973 INFO (qtp1900424130-6741) [n:127.0.0.1:33495_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:33495_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=3350
[junit4] 2> 2000646 INFO (OverseerCollectionConfigSetProcessor-98900015091154949-127.0.0.1:33900_-n_0000000000) [n:127.0.0.1:33900_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2000724 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-2-001 of type NRT
[junit4] 2> 2000725 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2000886 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@14b74cae{/,null,AVAILABLE}
[junit4] 2> 2000886 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@504f126f{SSL,[ssl, http/1.1]}{127.0.0.1:35892}
[junit4] 2> 2000886 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.Server Started @2017275ms
[junit4] 2> 2000886 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=35892, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-2-001/cores}
[junit4] 2> 2000887 ERROR (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2000887 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 2000887 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2000887 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2000887 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-27T08:54:44.610Z
[junit4] 2> 2000975 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2000975 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-2-001/solr.xml
[junit4] 2> 2000979 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2000979 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2001012 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2001049 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33010/solr
[junit4] 2> 2001108 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:35892_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2001143 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:35892_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2001145 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:35892_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35892_
[junit4] 2> 2001145 INFO (zkCallback-792-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2001145 INFO (zkCallback-797-thread-1-processing-n:127.0.0.1:33495_) [n:127.0.0.1:33495_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2001158 INFO (zkCallback-784-thread-1-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2001216 INFO (zkCallback-803-thread-1-processing-n:127.0.0.1:35892_) [n:127.0.0.1:35892_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2002031 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:35892_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 2002127 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:35892_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 2002127 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:35892_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 2002129 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:35892_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-2-001/cores
[junit4] 2> 2002733 INFO (qtp1900424130-6734) [n:127.0.0.1:33495_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:35892_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2002735 INFO (OverseerThreadFactory-2236-thread-4-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:35892_ for creating new replica
[junit4] 2> 2003168 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n23&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2003168 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2004304 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2004466 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.s.IndexSchema [collection1_shard1_replica_n23] Schema name=test
[junit4] 2> 2005369 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2005408 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n23' using configuration from collection collection1, trusted=true
[junit4] 2> 2005408 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n23' (registry 'solr.core.collection1.shard1.replica_n23') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 2005408 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2005408 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.SolrCore [[collection1_shard1_replica_n23] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-2-001/cores/collection1_shard1_replica_n23], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-2-001/cores/collection1_shard1_replica_n23/data/]
[junit4] 2> 2005444 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=32.7412109375, floorSegmentMB=0.9931640625, forceMergeDeletesPctAllowed=4.645882138600275, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4070763039360047
[junit4] 2> 2005508 WARN (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2005984 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2005984 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2005985 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2005985 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2005986 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=16.9990234375, floorSegmentMB=1.7177734375, forceMergeDeletesPctAllowed=29.80170751436371, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.25869791656211993
[junit4] 2> 2006021 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.s.SolrIndexSearcher Opening [Searcher@cd9f940[collection1_shard1_replica_n23] main]
[junit4] 2> 2006022 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2006023 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2006057 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2006059 INFO (searcherExecutor-2261-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.SolrCore [collection1_shard1_replica_n23] Registered new searcher Searcher@cd9f940[collection1_shard1_replica_n23] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2006059 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582400263717650432
[junit4] 2> 2006116 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.ZkController Core needs to recover:collection1_shard1_replica_n23
[junit4] 2> 2006116 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n23&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2997
[junit4] 2> 2006131 INFO (qtp1900424130-6734) [n:127.0.0.1:33495_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:35892_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=3398
[junit4] 2> 2006133 INFO (updateExecutor-800-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2006133 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2006194 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2006195 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard1_replica_n23]
[junit4] 2> 2006195 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2006195 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard1_replica_n23] as recovering, leader is [https://127.0.0.1:33495/collection1_shard1_replica_n21/] and I am [https://127.0.0.1:35892/collection1_shard1_replica_n23/]
[junit4] 2> 2006236 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Sending prep recovery command to [https://127.0.0.1:33495]; [WaitForState: action=PREPRECOVERY&core=collection1_shard1_replica_n21&nodeName=127.0.0.1:35892_&coreNodeName=core_node24&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2006272 INFO (qtp1900424130-6738) [n:127.0.0.1:33495_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node24, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, maxTime: 183 s
[junit4] 2> 2006272 INFO (qtp1900424130-6738) [n:127.0.0.1:33495_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1_shard1_replica_n21, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:35892_, coreNodeName=core_node24, onlyIfActiveCheckResult=false, nodeProps: core_node24:{"core":"collection1_shard1_replica_n23","base_url":"https://127.0.0.1:35892","node_name":"127.0.0.1:35892_","state":"recovering","type":"NRT"}
[junit4] 2> 2006272 INFO (qtp1900424130-6738) [n:127.0.0.1:33495_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node24, state: recovering, checkLive: true, onlyIfLeader: true for: 0 seconds.
[junit4] 2> 2006272 INFO (qtp1900424130-6738) [n:127.0.0.1:33495_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:35892_&onlyIfLeaderActive=true&core=collection1_shard1_replica_n21&coreNodeName=core_node24&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2006777 INFO (OverseerCollectionConfigSetProcessor-98900015091154949-127.0.0.1:33900_-n_0000000000) [n:127.0.0.1:33900_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2006971 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [https://127.0.0.1:33495/collection1_shard1_replica_n21/] - recoveringAfterStartup=[true]
[junit4] 2> 2006972 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_n23 url=https://127.0.0.1:35892 START replicas=[https://127.0.0.1:33495/collection1_shard1_replica_n21/] nUpdates=1000
[junit4] 2> 2007102 INFO (qtp1900424130-6739) [n:127.0.0.1:33495_ c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_n21] o.a.s.u.IndexFingerprint IndexFingerprint millis:56.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2007102 INFO (qtp1900424130-6739) [n:127.0.0.1:33495_ c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_n21] o.a.s.c.S.Request [collection1_shard1_replica_n21] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=86
[junit4] 2> 2007119 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2007119 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2007119 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2007119 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2007191 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2007191 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2007191 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2007191 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2007192 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2007268 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-3-001 of type NRT
[junit4] 2> 2007317 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2007427 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@20a6a8d0{/,null,AVAILABLE}
[junit4] 2> 2007430 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@76aeb631{SSL,[ssl, http/1.1]}{127.0.0.1:41808}
[junit4] 2> 2007430 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.Server Started @2023819ms
[junit4] 2> 2007430 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=41808, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-3-001/cores}
[junit4] 2> 2007430 ERROR (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2007447 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 2007448 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2007448 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2007448 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-27T08:54:51.171Z
[junit4] 2> 2007533 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2007533 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-3-001/solr.xml
[junit4] 2> 2007536 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2007536 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2007570 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2007628 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33010/solr
[junit4] 2> 2007685 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:41808_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2007686 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:41808_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2007687 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:41808_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41808_
[junit4] 2> 2007720 INFO (zkCallback-784-thread-1-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2007793 INFO (zkCallback-810-thread-1-processing-n:127.0.0.1:41808_) [n:127.0.0.1:41808_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2007793 INFO (zkCallback-803-thread-1-processing-n:127.0.0.1:35892_) [n:127.0.0.1:35892_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2007720 INFO (zkCallback-792-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2007821 INFO (zkCallback-797-thread-1-processing-n:127.0.0.1:33495_) [n:127.0.0.1:33495_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2008401 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:41808_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 2008563 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:41808_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 2008563 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:41808_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 2008564 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:41808_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-3-001/cores
[junit4] 2> 2008856 INFO (qtp1900424130-6740) [n:127.0.0.1:33495_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41808_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2008892 INFO (OverseerThreadFactory-2236-thread-5-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:41808_ for creating new replica
[junit4] 2> 2009207 INFO (qtp950198755-6802) [n:127.0.0.1:41808_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n25&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2009207 INFO (qtp950198755-6802) [n:127.0.0.1:41808_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2010447 INFO (qtp950198755-6802) [n:127.0.0.1:41808_ c:collection1 s:shard1 x:collection1_shard1_replica_n25] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2010722 INFO (qtp950198755-6802) [n:127.0.0.1:41808_ c:collection1 s:shard1 x:collection1_shard1_rep
[...truncated too long message...]
ng() + ": " + e.getMessage(), e);
[ecj-lint] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: 'client' is not closed at this location
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 11. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java (at line 204)
[ecj-lint] Analyzer a1 = new WhitespaceAnalyzer();
[ecj-lint] ^^
[ecj-lint] Resource leak: 'a1' is never closed
[ecj-lint] ----------
[ecj-lint] 12. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java (at line 207)
[ecj-lint] OffsetWindowTokenFilter tots = new OffsetWindowTokenFilter(tokenStream);
[ecj-lint] ^^^^
[ecj-lint] Resource leak: 'tots' is never closed
[ecj-lint] ----------
[ecj-lint] 13. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java (at line 211)
[ecj-lint] Analyzer a2 = new WhitespaceAnalyzer();
[ecj-lint] ^^
[ecj-lint] Resource leak: 'a2' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 14. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/response/TestJavabinTupleStreamParser.java (at line 72)
[ecj-lint] JavabinTupleStreamParser parser = new JavabinTupleStreamParser(new ByteArrayInputStream(bytes), true);
[ecj-lint] ^^^^^^
[ecj-lint] Resource leak: 'parser' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 15. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/search/TestDocSet.java (at line 243)
[ecj-lint] return loadfactor!=0 ? new HashDocSet(a,0,n,1/loadfactor) : new HashDocSet(a,0,n);
[ecj-lint] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
[ecj-lint] ----------
[ecj-lint] 16. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/search/TestDocSet.java (at line 528)
[ecj-lint] DocSet a = new BitDocSet(bs);
[ecj-lint] ^
[ecj-lint] Resource leak: 'a' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 17. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/security/TestPKIAuthenticationPlugin.java (at line 76)
[ecj-lint] final MockPKIAuthenticationPlugin mock = new MockPKIAuthenticationPlugin(null, nodeName);
[ecj-lint] ^^^^
[ecj-lint] Resource leak: 'mock' is never closed
[ecj-lint] ----------
[ecj-lint] 18. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/security/TestPKIAuthenticationPlugin.java (at line 131)
[ecj-lint] MockPKIAuthenticationPlugin mock1 = new MockPKIAuthenticationPlugin(null, nodeName) {
[ecj-lint] ^^^^^
[ecj-lint] Resource leak: 'mock1' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 19. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/security/TestRuleBasedAuthorizationPlugin.java (at line 380)
[ecj-lint] RuleBasedAuthorizationPlugin plugin = new RuleBasedAuthorizationPlugin();
[ecj-lint] ^^^^^^
[ecj-lint] Resource leak: 'plugin' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 20. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/security/TestSha256AuthenticationProvider.java (at line 49)
[ecj-lint] BasicAuthPlugin basicAuthPlugin = new BasicAuthPlugin();
[ecj-lint] ^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: 'basicAuthPlugin' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 21. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java (at line 42)
[ecj-lint] WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer();
[ecj-lint] ^^^^^^^^
[ecj-lint] Resource leak: 'analyzer' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 22. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java (at line 139)
[ecj-lint] IndexWriter w = new IndexWriter(d, newIndexWriterConfig(analyzer));
[ecj-lint] ^
[ecj-lint] Resource leak: 'w' is never closed
[ecj-lint] ----------
[ecj-lint] 23. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java (at line 172)
[ecj-lint] throw iae;
[ecj-lint] ^^^^^^^^^^
[ecj-lint] Resource leak: 'w' is not closed at this location
[ecj-lint] ----------
[ecj-lint] 24. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java (at line 178)
[ecj-lint] return;
[ecj-lint] ^^^^^^^
[ecj-lint] Resource leak: 'w' is not closed at this location
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 25. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java (at line 134)
[ecj-lint] SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
[ecj-lint] ^^^^^^^^^^
[ecj-lint] Resource leak: 'cmdDistrib' is never closed
[ecj-lint] ----------
[ecj-lint] 26. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java (at line 333)
[ecj-lint] SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 5, 0);
[ecj-lint] ^^^^^^^^^^
[ecj-lint] Resource leak: 'cmdDistrib' is never closed
[ecj-lint] ----------
[ecj-lint] 27. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java (at line 367)
[ecj-lint] SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 5, 0);
[ecj-lint] ^^^^^^^^^^
[ecj-lint] Resource leak: 'cmdDistrib' is never closed
[ecj-lint] ----------
[ecj-lint] 28. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java (at line 413)
[ecj-lint] SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 5, 0);
[ecj-lint] ^^^^^^^^^^
[ecj-lint] Resource leak: 'cmdDistrib' is never closed
[ecj-lint] ----------
[ecj-lint] 29. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java (at line 458)
[ecj-lint] SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
[ecj-lint] ^^^^^^^^^^
[ecj-lint] Resource leak: 'cmdDistrib' is never closed
[ecj-lint] ----------
[ecj-lint] 30. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java (at line 516)
[ecj-lint] SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
[ecj-lint] ^^^^^^^^^^
[ecj-lint] Resource leak: 'cmdDistrib' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 31. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java (at line 181)
[ecj-lint] EmbeddedSolrServer server1 = new EmbeddedSolrServer(h.getCoreContainer(), "split1");
[ecj-lint] ^^^^^^^
[ecj-lint] Resource leak: 'server1' is never closed
[ecj-lint] ----------
[ecj-lint] 32. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java (at line 182)
[ecj-lint] EmbeddedSolrServer server2 = new EmbeddedSolrServer(h.getCoreContainer(), "split2");
[ecj-lint] ^^^^^^^
[ecj-lint] Resource leak: 'server2' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 33. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/processor/RecordingUpdateProcessorFactory.java (at line 67)
[ecj-lint] return recording ? new RecordingUpdateRequestProcessor(commandQueue, next) : next;
[ecj-lint] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
[ecj-lint] ----------
[ecj-lint] 33 problems (1 error, 32 warnings)
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:826: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:101: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build.xml:684: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:2093: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:2126: Compile failed; see the compiler error output for details.
Total time: 344 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 194 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/194/
6 tests failed.
FAILED: org.apache.solr.cloud.HttpPartitionTest.test
Error Message:
The partitioned replica did not get marked down expected:<[down]> but was:<[active]>
Stack Trace:
org.junit.ComparisonFailure: The partitioned replica did not get marked down expected:<[down]> but was:<[active]>
at __randomizedtesting.SeedInfo.seed([4D0F748AACCA44B0:C55B4B5002362948]:0)
at org.junit.Assert.assertEquals(Assert.java:125)
at org.apache.solr.cloud.HttpPartitionTest.testMinRf(HttpPartitionTest.java:241)
at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:127)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestCloudRecovery.leaderRecoverFromLogOnStartupTest
Error Message:
Stack Trace:
java.lang.NullPointerException
at __randomizedtesting.SeedInfo.seed([4D0F748AACCA44B0:39FF95CFE8EE033F]:0)
at org.apache.solr.cloud.TestCloudRecovery.leaderRecoverFromLogOnStartupTest(TestCloudRecovery.java:124)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration
Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:50483_solr wasn't created
Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:50483_solr wasn't created
at __randomizedtesting.SeedInfo.seed([4D0F748AACCA44B0:55B5FC86A2FF895F]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration(TriggerIntegrationTest.java:909)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.handler.TestSQLHandler.doTest
Error Message:
Error from server at http://127.0.0.1:43468: KeeperErrorCode = Session expired for /overseer/collection-queue-work/qnr-
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:43468: KeeperErrorCode = Session expired for /overseer/collection-queue-work/qnr-
at __randomizedtesting.SeedInfo.seed([4D0F748AACCA44B0:EA4BCC2EC1715709]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.reloadCollection(AbstractFullDistribZkTestBase.java:2045)
at org.apache.solr.handler.TestSQLHandler.testBasicSelect(TestSQLHandler.java:306)
at org.apache.solr.handler.TestSQLHandler.doTest(TestSQLHandler.java:82)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory
Error Message:
replica never fully recovered
Stack Trace:
java.lang.AssertionError: replica never fully recovered
at __randomizedtesting.SeedInfo.seed([4D0F748AACCA44B0:20F3D0771682BBB7]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.waitForRecovery(AutoscalingHistoryHandlerTest.java:303)
at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:255)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.search.join.BlockJoinFacetDistribTest.testBJQFacetComponent
Error Message:
Could not load collection from ZK: facetcollection
Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: facetcollection
at __randomizedtesting.SeedInfo.seed([4D0F748AACCA44B0:1DF2C4F727369138]:0)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1188)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:832)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
at org.apache.solr.search.join.BlockJoinFacetDistribTest.testBJQFacetComponent(BlockJoinFacetDistribTest.java:91)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/facetcollection/state.json
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
... 47 more
Build Log:
[...truncated 12564 lines...]
[junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/init-core-data-001
[junit4] 2> 3334387 WARN (SUITE-HttpPartitionTest-seed#[4D0F748AACCA44B0]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=24 numCloses=24
[junit4] 2> 3334387 INFO (SUITE-HttpPartitionTest-seed#[4D0F748AACCA44B0]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 3334524 INFO (SUITE-HttpPartitionTest-seed#[4D0F748AACCA44B0]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 3334524 INFO (SUITE-HttpPartitionTest-seed#[4D0F748AACCA44B0]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 3334524 INFO (SUITE-HttpPartitionTest-seed#[4D0F748AACCA44B0]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 3334532 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3334576 INFO (Thread-1783) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3334576 INFO (Thread-1783) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3334632 ERROR (Thread-1783) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3334708 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.ZkTestServer start zk server on port:43744
[junit4] 2> 3334762 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f5a42748a0000, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3335269 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 3335510 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 3335571 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 3335588 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 3335592 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 3335596 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 3335600 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 3335608 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 3335609 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 3335609 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 3335610 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 3335620 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
[junit4] 2> 3336750 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3336753 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@23c008fd{/,null,AVAILABLE}
[junit4] 2> 3336753 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@27375f42{HTTP/1.1,[http/1.1]}{127.0.0.1:52638}
[junit4] 2> 3336753 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.Server Started @3358194ms
[junit4] 2> 3336753 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/tempDir-001/control/data, hostContext=/, hostPort=59675, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/control-001/cores}
[junit4] 2> 3336754 ERROR (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3336754 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 3336754 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3336754 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 3336754 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-26T19:57:07.712Z
[junit4] 2> 3336970 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3336971 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/control-001/solr.xml
[junit4] 2> 3336982 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3336982 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3336983 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3337033 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43744/solr
[junit4] 2> 3339264 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:59675_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3339268 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:59675_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59675_
[junit4] 2> 3339276 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:59675_ ] o.a.s.c.Overseer Overseer (id=98896958294982660-127.0.0.1:59675_-n_0000000000) starting
[junit4] 2> 3339620 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:59675_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59675_
[junit4] 2> 3339624 INFO (zkCallback-1153-thread-1-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3340493 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:59675_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3340595 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:59675_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3340596 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:59675_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3340675 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:59675_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/control-001/cores
[junit4] 2> 3342438 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3342438 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43744/solr ready
[junit4] 2> 3342457 INFO (SocketProxy-Acceptor-59675) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=54939,localport=59675], receiveBufferSize:531000
[junit4] 2> 3342552 INFO (SocketProxy-Acceptor-59675) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=52638,localport=48610], receiveBufferSize=530904
[junit4] 2> 3342700 INFO (qtp1321684759-7417) [n:127.0.0.1:59675_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:59675_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3342835 INFO (OverseerThreadFactory-3201-thread-1-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 3342985 INFO (SocketProxy-Acceptor-59675) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=54941,localport=59675], receiveBufferSize:531000
[junit4] 2> 3343000 INFO (SocketProxy-Acceptor-59675) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=52638,localport=48612], receiveBufferSize=530904
[junit4] 2> 3343073 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3343073 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3343372 INFO (zkCallback-1153-thread-1-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3344374 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 3344719 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 3345835 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3346017 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 3346017 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3346081 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3346081 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 3346264 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=30.1689453125, floorSegmentMB=1.142578125, forceMergeDeletesPctAllowed=24.11397041105384, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24045656329513784
[junit4] 2> 3346270 WARN (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3348508 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3348508 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3348517 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3348517 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3348611 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=96.0751953125, floorSegmentMB=0.677734375, forceMergeDeletesPctAllowed=27.557608195764637, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3967483425731648
[junit4] 2> 3348611 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@65ddea9a[control_collection_shard1_replica_n1] main]
[junit4] 2> 3348612 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3348803 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3348804 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3348807 INFO (searcherExecutor-3204-thread-1-processing-n:127.0.0.1:59675_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@65ddea9a[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3348807 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582351347629424640
[junit4] 2> 3348820 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3348820 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3348820 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:59675/control_collection_shard1_replica_n1/
[junit4] 2> 3348820 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3348820 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:59675/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 3348820 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3348824 INFO (zkCallback-1153-thread-2-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3348824 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:59675/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 3348825 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3348826 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=5752
[junit4] 2> 3348830 INFO (qtp1321684759-7417) [n:127.0.0.1:59675_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3348954 INFO (zkCallback-1153-thread-2-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3349240 INFO (OverseerCollectionConfigSetProcessor-98896958294982660-127.0.0.1:59675_-n_0000000000) [n:127.0.0.1:59675_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3349838 INFO (qtp1321684759-7417) [n:127.0.0.1:59675_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:59675_&wt=javabin&version=2} status=0 QTime=7137
[junit4] 2> 3352201 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3352202 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43744/solr ready
[junit4] 2> 3352202 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 3352202 INFO (SocketProxy-Acceptor-59675) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=54947,localport=59675], receiveBufferSize:531000
[junit4] 2> 3352282 INFO (SocketProxy-Acceptor-59675) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=52638,localport=48618], receiveBufferSize=530904
[junit4] 2> 3352322 INFO (qtp1321684759-7418) [n:127.0.0.1:59675_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3352449 INFO (OverseerThreadFactory-3201-thread-2-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 3352450 WARN (OverseerThreadFactory-3201-thread-2-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 3352886 INFO (qtp1321684759-7418) [n:127.0.0.1:59675_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3352886 INFO (qtp1321684759-7418) [n:127.0.0.1:59675_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=563
[junit4] 2> 3353607 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-1-001 of type TLOG
[junit4] 2> 3353608 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3354396 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1fb6a4d7{/,null,AVAILABLE}
[junit4] 2> 3354396 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3583ebd{HTTP/1.1,[http/1.1]}{127.0.0.1:39669}
[junit4] 2> 3354396 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.Server Started @3375836ms
[junit4] 2> 3354396 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=56283, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-1-001/cores}
[junit4] 2> 3354397 ERROR (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3354397 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 3354397 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3354397 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 3354397 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-26T19:57:25.355Z
[junit4] 2> 3354451 INFO (OverseerCollectionConfigSetProcessor-98896958294982660-127.0.0.1:59675_-n_0000000000) [n:127.0.0.1:59675_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3357976 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3357976 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-1-001/solr.xml
[junit4] 2> 3358013 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3358013 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3358015 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3358136 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43744/solr
[junit4] 2> 3359143 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:56283_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3359144 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:56283_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3359146 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:56283_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56283_
[junit4] 2> 3359374 INFO (zkCallback-1166-thread-2-processing-n:127.0.0.1:56283_) [n:127.0.0.1:56283_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3359620 INFO (zkCallback-1153-thread-2-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3359745 INFO (zkCallback-1161-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3359963 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:56283_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3360111 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:56283_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3360111 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:56283_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3360279 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:56283_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-1-001/cores
[junit4] 2> 3360528 INFO (SocketProxy-Acceptor-56283) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=39961,localport=56283], receiveBufferSize:531000
[junit4] 2> 3360608 INFO (SocketProxy-Acceptor-56283) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=39669,localport=42583], receiveBufferSize=530904
[junit4] 2> 3360718 INFO (qtp1771810288-7471) [n:127.0.0.1:56283_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:56283_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3360745 INFO (OverseerThreadFactory-3201-thread-3-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:56283_ for creating new replica
[junit4] 2> 3360748 INFO (SocketProxy-Acceptor-56283) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=39963,localport=56283], receiveBufferSize:531000
[junit4] 2> 3360749 INFO (SocketProxy-Acceptor-56283) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=39669,localport=42585], receiveBufferSize=530904
[junit4] 2> 3360793 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 3360793 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3360939 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 3361071 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema [collection1_shard2_replica_t41] Schema name=test
[junit4] 2> 3361962 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3361973 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t41' using configuration from collection collection1, trusted=true
[junit4] 2> 3361974 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t41' (registry 'solr.core.collection1.shard2.replica_t41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3361974 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3361974 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [[collection1_shard2_replica_t41] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-1-001/cores/collection1_shard2_replica_t41], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-1-001/cores/collection1_shard2_replica_t41/data/]
[junit4] 2> 3361976 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=30.1689453125, floorSegmentMB=1.142578125, forceMergeDeletesPctAllowed=24.11397041105384, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24045656329513784
[junit4] 2> 3361978 WARN (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3362010 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3362010 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3362011 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3362011 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3362013 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=96.0751953125, floorSegmentMB=0.677734375, forceMergeDeletesPctAllowed=27.557608195764637, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3967483425731648
[junit4] 2> 3362013 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.SolrIndexSearcher Opening [Searcher@53efa664[collection1_shard2_replica_t41] main]
[junit4] 2> 3362014 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3362014 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3362014 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3362015 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582351361479016448
[junit4] 2> 3362018 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3362018 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3362018 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56283/collection1_shard2_replica_t41/
[junit4] 2> 3362018 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3362018 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy http://127.0.0.1:56283/collection1_shard2_replica_t41/ has no replicas
[junit4] 2> 3362018 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3362019 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController collection1_shard2_replica_t41 stopping background replication from leader
[junit4] 2> 3362021 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56283/collection1_shard2_replica_t41/ shard2
[junit4] 2> 3362022 INFO (searcherExecutor-3215-thread-1-processing-n:127.0.0.1:56283_ x:collection1_shard2_replica_t41 s:shard2 c:collection1) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [collection1_shard2_replica_t41] Registered new searcher Searcher@53efa664[collection1_shard2_replica_t41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3362172 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3362173 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1379
[junit4] 2> 3362175 INFO (qtp1771810288-7471) [n:127.0.0.1:56283_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:56283_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=1456
[junit4] 2> 3362748 INFO (OverseerCollectionConfigSetProcessor-98896958294982660-127.0.0.1:59675_-n_0000000000) [n:127.0.0.1:59675_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3363089 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-2-001 of type TLOG
[junit4] 2> 3363090 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3363416 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2042af88{/,null,AVAILABLE}
[junit4] 2> 3363417 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@34d81f48{HTTP/1.1,[http/1.1]}{127.0.0.1:41685}
[junit4] 2> 3363417 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.Server Started @3384857ms
[junit4] 2> 3363417 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=54481, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-2-001/cores}
[junit4] 2> 3363417 ERROR (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3363424 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 3363424 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3363424 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 3363424 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-26T19:57:34.382Z
[junit4] 2> 3363611 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3363611 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-2-001/solr.xml
[junit4] 2> 3363614 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3363615 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3363616 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3363622 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43744/solr
[junit4] 2> 3364972 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:54481_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3364974 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:54481_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3364989 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:54481_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54481_
[junit4] 2> 3365009 INFO (zkCallback-1153-thread-1-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3365009 INFO (zkCallback-1161-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3365009 INFO (zkCallback-1166-thread-2-processing-n:127.0.0.1:56283_) [n:127.0.0.1:56283_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3365226 INFO (zkCallback-1172-thread-1-processing-n:127.0.0.1:54481_) [n:127.0.0.1:54481_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3366416 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:54481_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3366673 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:54481_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3366673 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:54481_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3366674 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:54481_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-2-001/cores
[junit4] 2> 3367061 INFO (SocketProxy-Acceptor-54481) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=33414,localport=54481], receiveBufferSize:531000
[junit4] 2> 3367129 INFO (SocketProxy-Acceptor-54481) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=41685,localport=50630], receiveBufferSize=530904
[junit4] 2> 3367161 INFO (qtp239820775-7507) [n:127.0.0.1:54481_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:54481_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3367461 INFO (OverseerThreadFactory-3201-thread-4-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:54481_ for creating new replica
[junit4] 2> 3367512 INFO (SocketProxy-Acceptor-54481) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=33416,localport=54481], receiveBufferSize:531000
[junit4] 2> 3367512 INFO (SocketProxy-Acceptor-54481) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=41685,localport=50632], receiveBufferSize=530904
[junit4] 2> 3367591 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 3367591 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3368781 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 3368932 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema [collection1_shard1_replica_t43] Schema name=test
[junit4] 2> 3369891 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3370028 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t43' using configuration from collection collection1, trusted=true
[junit4] 2> 3370029 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t43' (registry 'solr.core.collection1.shard1.replica_t43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3370029 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3370029 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [[collection1_shard1_replica_t43] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-2-001/cores/collection1_shard1_replica_t43], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-2-001/cores/collection1_shard1_replica_t43/data/]
[junit4] 2> 3370076 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=30.1689453125, floorSegmentMB=1.142578125, forceMergeDeletesPctAllowed=24.11397041105384, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24045656329513784
[junit4] 2> 3370118 WARN (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3372388 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3372388 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3372453 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3372453 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3372480 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=96.0751953125, floorSegmentMB=0.677734375, forceMergeDeletesPctAllowed=27.557608195764637, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3967483425731648
[junit4] 2> 3372480 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.SolrIndexSearcher Opening [Searcher@29b39e83[collection1_shard1_replica_t43] main]
[junit4] 2> 3372481 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3372481 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3372482 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3372482 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582351372454461440
[junit4] 2> 3372493 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3372493 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3372493 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:54481/collection1_shard1_replica_t43/
[junit4] 2> 3372493 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3372493 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy http://127.0.0.1:54481/collection1_shard1_replica_t43/ has no replicas
[junit4] 2> 3372494 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3372504 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController collection1_shard1_replica_t43 stopping background replication from leader
[junit4] 2> 3372512 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:54481/collection1_shard1_replica_t43/ shard1
[junit4] 2> 3372512 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3372512 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController No registered searcher found for core: collection1_shard1_replica_t43, waiting until a searcher is registered before publishing as active
[junit4] 2> 3372528 INFO (searcherExecutor-3226-thread-1-processing-n:127.0.0.1:54481_ x:collection1_shard1_replica_t43 s:shard1 c:collection1) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [collection1_shard1_replica_t43] Registered new searcher Searcher@29b39e83[collection1_shard1_replica_t43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3372529 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController Found a registered searcher: Searcher@29b39e83[collection1_shard1_replica_t43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}, took: 16.0 ms for core: collection1_shard1_replica_t43, id: org.apache.solr.core.SolrCore@453bc184
[junit4] 2> 3372588 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=4997
[junit4] 2> 3372590 INFO (qtp239820775-7507) [n:127.0.0.1:54481_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:54481_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=5429
[junit4] 2> 3374524 INFO (OverseerCollectionConfigSetProcessor-98896958294982660-127.0.0.1:59675_-n_0000000000) [n:127.0.0.1:59675_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3375177 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-3-001 of type TLOG
[junit4] 2> 3375178 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3375198 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@61c1d54a{/,null,AVAILABLE}
[junit4] 2> 3375198 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@d946d3f{HTTP/1.1,[http/1.1]}{127.0.0.1:49205}
[junit4] 2> 3375198 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.Server Started @3396638ms
[junit4] 2> 3375198 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/tempDir-001/jetty3, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=60243, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-3-001/cores}
[junit4] 2> 3375198 ERROR (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3375216 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 3375216 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3375216 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 3375216 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-26T19:57:46.174Z
[junit4] 2> 3375457 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3375457 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-00
[...truncated too long message...]
.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 10661078 INFO (zkCallback-4667-thread-2-processing-n:127.0.0.1:49106_solr) [n:127.0.0.1:49106_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:49106_solr
[junit4] 2> 10661097 INFO (jetty-closer-4643-thread-5) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7fff2926{/solr,null,UNAVAILABLE}
[junit4] 2> 10661194 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f5ab19a7e000b, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 10661195 INFO (zkCallback-4674-thread-2-processing-n:127.0.0.1:51315_solr) [n:127.0.0.1:51315_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:51315_solr
[junit4] 2> 10661238 INFO (coreCloseExecutor-10138-thread-1) [n:127.0.0.1:41606_solr c:facetcollection s:shard1 r:core_node2 x:facetcollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.facetcollection.shard1.leader, tag=1482760307
[junit4] 2> 10661241 INFO (coreCloseExecutor-10139-thread-1) [n:127.0.0.1:43360_solr c:facetcollection s:shard3 r:core_node12 x:facetcollection_shard3_replica_n11] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.facetcollection.shard3.leader, tag=496671074
[junit4] 2> 10661241 INFO (coreCloseExecutor-10137-thread-1) [n:127.0.0.1:34403_solr c:facetcollection s:shard2 r:core_node8 x:facetcollection_shard2_replica_n7] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.facetcollection.shard2.leader, tag=531754703
[junit4] 2> 10661241 INFO (coreCloseExecutor-10136-thread-1) [n:127.0.0.1:51315_solr c:facetcollection s:shard2 r:core_node6 x:facetcollection_shard2_replica_n5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.facetcollection.shard2.leader, tag=616504417
[junit4] 2> 10661410 INFO (jetty-closer-4643-thread-6) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@14b5e0d1{/solr,null,UNAVAILABLE}
[junit4] 2> 10661477 INFO (jetty-closer-4643-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4e938237{/solr,null,UNAVAILABLE}
[junit4] 2> 10661477 INFO (jetty-closer-4643-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@567c3434{/solr,null,UNAVAILABLE}
[junit4] 2> 10662627 INFO (jetty-closer-4643-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6229bb0c{/solr,null,UNAVAILABLE}
[junit4] 2> 10662973 INFO (jetty-closer-4643-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@45ade1de{/solr,null,UNAVAILABLE}
[junit4] 2> 10662989 ERROR (SUITE-BlockJoinFacetDistribTest-seed#[4D0F748AACCA44B0]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 10662989 INFO (SUITE-BlockJoinFacetDistribTest-seed#[4D0F748AACCA44B0]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46876 46876
[junit4] 2> 10663109 INFO (Thread-8278) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46876 46876
[junit4] 2> 10663110 WARN (Thread-8278) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/aliases.json
[junit4] 2> 8 /solr/clusterprops.json
[junit4] 2> 6 /solr/security.json
[junit4] 2> 6 /solr/configs/solrCloudCollectionConfig
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 91 /solr/collections/facetcollection/state.json
[junit4] 2> 8 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/live_nodes
[junit4] 2> 8 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.search.join.BlockJoinFacetDistribTest_4D0F748AACCA44B0-001
[junit4] 2> Oct 26, 2017 9:59:11 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1838, maxMBSortInHeap=6.924484135782889, sim=RandomSimilarity(queryNorm=false): {}, locale=hi-IN, timezone=Europe/Mariehamn
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=248313664,total=526385152
[junit4] 2> NOTE: All tests run in this JVM: [TestJavabinTupleStreamParser, TestNumericTerms32, OverseerRolesTest, CloudMLTQParserTest, BadCopyFieldTest, TestHighlightDedupGrouping, MultiTermTest, QueryParsingTest, TestReplicationHandler, ResponseLogComponentTest, MoveReplicaHDFSFailoverTest, DateFieldTest, BaseCdcrDistributedZkTest, TestSolrCloudWithKerberosAlt, TestAuthorizationFramework, HttpTriggerListenerTest, TestIBSimilarityFactory, SolrJmxReporterCloudTest, SliceStateTest, HdfsNNFailoverTest, DeleteReplicaTest, GraphQueryTest, SolrCoreTest, SolrCloudReportersTest, TestSolrCLIRunExample, SyncSliceTest, DistributedQueryComponentCustomSortTest, TestRestManager, TestUtils, TestSchemaVersionResource, DistributedQueueTest, HLLSerializationTest, VersionInfoTest, ResourceLoaderTest, TestNoOpRegenerator, TestLuceneIndexBackCompat, CollectionStateFormat2Test, DirectUpdateHandlerOptimizeTest, CoreAdminCreateDiscoverTest, BitVectorTest, TestManagedStopFilterFactory, CursorPagingTest, TestSolrDeletionPolicy1, TestPrepRecovery, TestCrossCoreJoin, TestHashQParserPlugin, TestBinaryField, TestMaxScoreQueryParser, DocValuesTest, RAMDirectoryFactoryTest, TestOrdValues, NoCacheHeaderTest, BasicAuthIntegrationTest, CurrencyFieldTypeTest, TestLockTree, AddBlockUpdateTest, TestRecovery, FileUtilsTest, IndexSchemaRuntimeFieldTest, TestMaxTokenLenTokenizer, TestEmbeddedSolrServerSchemaAPI, TestAuthenticationFramework, TestSchemaNameResource, TestDocBasedVersionConstraints, TestTrieFacet, ShardSplitTest, TestSweetSpotSimilarityFactory, TestReversedWildcardFilterFactory, TestAnalyzeInfixSuggestions, AutoAddReplicasIntegrationTest, TestHalfAndHalfDocValues, TestCodecSupport, LeaderInitiatedRecoveryOnShardRestartTest, MinimalSchemaTest, DistributedMLTComponentTest, TermVectorComponentTest, TestExactSharedStatsCache, BlockDirectoryTest, HdfsRestartWhileUpdatingTest, TestRawTransformer, TestComplexPhraseLeadingWildcard, FacetPivotSmallTest, TestSolrFieldCacheBean, DistribCursorPagingTest, TestLeaderElectionZkExpiry, SimpleFacetsTest, TestSolrIndexConfig, TestReplicaProperties, TestCloudManagedSchema, TestCoreAdminApis, LeaderElectionIntegrationTest, TestMergePolicyConfig, TestJoin, TestFoldingMultitermQuery, TestPHPSerializedResponseWriter, EnumFieldTest, TriLevelCompositeIdRoutingTest, PeerSyncReplicationTest, TestFieldResource, SuggestComponentTest, ClassificationUpdateProcessorTest, TestSha256AuthenticationProvider, TestDeleteCollectionOnDownNodes, SolrCoreMetricManagerTest, JsonLoaderTest, OverriddenZkACLAndCredentialsProvidersTest, QueryResultKeyTest, SolrCLIZkUtilsTest, DateMathParserTest, BasicFunctionalityTest, CleanupOldIndexTest, StatelessScriptUpdateProcessorFactoryTest, TestCorePropertiesReload, TestCSVLoader, SpellCheckCollatorTest, ZkControllerTest, BasicDistributedZkTest, SearchRateTriggerTest, SearchHandlerTest, TestFreeTextSuggestions, CloudExitableDirectoryReaderTest, CollectionsAPIAsyncDistributedZkTest, CustomCollectionTest, DeleteStatusTest, DocValuesNotIndexedTest, LeaderFailureAfterFreshStartTest, LeaderInitiatedRecoveryOnCommitTest, NodeMutatorTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerStatusTest, RecoveryAfterSoftCommitTest, SaslZkACLProviderTest, SolrXmlInZkTest, TestCloudDeleteByQuery, TestCloudPseudoReturnFields, TestClusterProperties, TestCollectionAPI, TestConfigSetsAPIExclusivity, TestDistribDocBasedVersion, TestExclusionRuleCollectionAccess, TestHdfsCloudBackupRestore, TestRebalanceLeaders, TestSSLRandomization, TestSolrCloudWithSecureImpersonation, TestStressInPlaceUpdates, HdfsBasicDistributedZk2Test, HdfsChaosMonkeyNothingIsSafeTest, HdfsCollectionsAPIDistributedZkTest, HdfsRecoveryZkTest, RuleEngineTest, RulesTest, TestConfigSets, TestImplicitCoreProperties, TestInfoStreamLogging, TestShardHandlerFactory, TestSimpleTextCodec, TestSolrConfigHandler, TestSolrConfigHandlerCloud, DistributedFacetPivotLongTailTest, DistributedFacetPivotSmallTest, DistributedQueryComponentOptimizationTest, ShufflingReplicaListTransformerTest, SpatialHeatmapFacetsTest, TestHttpShardHandlerFactory, JavabinLoaderTest, TestSlowCompositeReaderWrapper, WrapperMergePolicyFactoryTest, CheckHdfsIndexTest, TestMacroExpander, SmileWriterTest, TestBinaryResponseWriter, TestExportWriter, TestSubQueryTransformer, TestManagedResourceStorage, TestDynamicFieldCollectionResource, TestSerializedLuceneMatchVersion, DateRangeFieldTest, SpatialRPTFieldTypeTest, TestManagedSchema, TestManagedSchemaThreadSafety, TestPointFields, TestInitQParser, TestPayloadCheckQParserPlugin, TestRandomCollapseQParserPlugin, TestMinMaxOnMultiValuedField, BlockJoinFacetDistribTest]
[junit4] Completed [700/744 (6!)] on J0 in 42.69s, 1 test, 1 error <<< FAILURES!
[...truncated 47385 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 193 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/193/
8 tests failed.
FAILED: org.apache.solr.cloud.DeleteReplicaTest.deleteReplicaAndVerifyDirectoryCleanup
Error Message:
No registered leader was found after waiting for 4000ms , collection: deletereplica_test slice: shard1 saw state=DocCollection(deletereplica_test//collections/deletereplica_test/state.json/9)={ "pullReplicas":"0", "replicationFactor":"2", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{"core_node3":{ "core":"deletereplica_test_shard1_replica_n1", "base_url":"https://127.0.0.1:58341/solr", "node_name":"127.0.0.1:58341_solr", "state":"active", "type":"NRT", "leader":"true"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"2", "tlogReplicas":"0"} with live_nodes=[127.0.0.1:50213_solr, 127.0.0.1:58341_solr, 127.0.0.1:55321_solr, 127.0.0.1:50473_solr]
Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 4000ms , collection: deletereplica_test slice: shard1 saw state=DocCollection(deletereplica_test//collections/deletereplica_test/state.json/9)={
"pullReplicas":"0",
"replicationFactor":"2",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{"core_node3":{
"core":"deletereplica_test_shard1_replica_n1",
"base_url":"https://127.0.0.1:58341/solr",
"node_name":"127.0.0.1:58341_solr",
"state":"active",
"type":"NRT",
"leader":"true"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"2",
"tlogReplicas":"0"} with live_nodes=[127.0.0.1:50213_solr, 127.0.0.1:58341_solr, 127.0.0.1:55321_solr, 127.0.0.1:50473_solr]
at __randomizedtesting.SeedInfo.seed([2BF42D8AD9FD7CA4:5B14A9305DB2D5AF]:0)
at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:816)
at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:800)
at org.apache.solr.cloud.DeleteReplicaTest.deleteReplicaAndVerifyDirectoryCleanup(DeleteReplicaTest.java:93)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestAuthenticationFramework.testBasics
Error Message:
Error from server at https://127.0.0.1:59320/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason: <pre> Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/> </body> </html>
Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:59320/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason:
<pre> Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>
at __randomizedtesting.SeedInfo.seed([2BF42D8AD9FD7CA4:162C83A6E11322D4]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:541)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:998)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
at org.apache.solr.cloud.TestAuthenticationFramework.collectionCreateSearchDeleteTwice(TestAuthenticationFramework.java:126)
at org.apache.solr.cloud.TestAuthenticationFramework.testBasics(TestAuthenticationFramework.java:74)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:59320/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason:
<pre> Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:590)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:518)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
FAILED: org.apache.solr.cloud.TestCloudPseudoReturnFields.testAugmentersGlobsExplicitAndScoreOhMyRTG
Error Message:
Error from server at https://127.0.0.1:51505/solr/org.apache.solr.cloud.TestCloudPseudoReturnFields_collection: no servers hosting shard: shard2
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:51505/solr/org.apache.solr.cloud.TestCloudPseudoReturnFields_collection: no servers hosting shard: shard2
at __randomizedtesting.SeedInfo.seed([2BF42D8AD9FD7CA4:76C9D67605608E]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.client.solrj.SolrClient.getById(SolrClient.java:1173)
at org.apache.solr.client.solrj.SolrClient.getById(SolrClient.java:1090)
at org.apache.solr.client.solrj.SolrClient.getById(SolrClient.java:1110)
at org.apache.solr.cloud.TestCloudPseudoReturnFields.testAugmentersGlobsExplicitAndScoreOhMyRTG(TestCloudPseudoReturnFields.java:779)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestCollectionAPI.test
Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:43790
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:43790
at __randomizedtesting.SeedInfo.seed([2BF42D8AD9FD7CA4:A3A012507701115C]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:637)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.TestCollectionAPI.testClusterStateMigration(TestCollectionAPI.java:645)
at org.apache.solr.cloud.TestCollectionAPI.test(TestCollectionAPI.java:94)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
at sun.security.ssl.InputRecord.read(InputRecord.java:503)
at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983)
at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:940)
at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
... 52 more
FAILED: org.apache.solr.cloud.TestSegmentSorting.testAtomicUpdateOfSegmentSortField
Error Message:
Error from server at http://127.0.0.1:35191/solr/testAtomicUpdateOfSegmentSortField_shard1_replica_n1: Cannot talk to ZooKeeper - Updates are disabled.
Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:35191/solr/testAtomicUpdateOfSegmentSortField_shard1_replica_n1: Cannot talk to ZooKeeper - Updates are disabled.
at __randomizedtesting.SeedInfo.seed([2BF42D8AD9FD7CA4:CAE60A8F0FB8E71C]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:541)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:998)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
at org.apache.solr.cloud.TestSegmentSorting.testAtomicUpdateOfSegmentSortField(TestSegmentSorting.java:140)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:47)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:35191/solr/testAtomicUpdateOfSegmentSortField_shard1_replica_n1: Cannot talk to ZooKeeper - Updates are disabled.
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:518)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
FAILED: org.apache.solr.handler.TestSolrConfigHandlerCloud.test
Error Message:
{ "responseHeader":{ "status":500, "QTime":32476}, "errorMessages":["4 out of 5 the property overlay to be of version 3 within 30 seconds! Failed cores: [https://127.0.0.1:56023/collection1_shard2_replica_n41/, https://127.0.0.1:54547/collection1_shard2_replica_n45/, https://127.0.0.1:52772/collection1_shard1_replica_n43/, https://127.0.0.1:50133/collection1_shard1_replica_n47/]\n"], "WARNING":"This response format is experimental. It is likely to change in the future.", "error":{ "metadata":[ "error-class","org.apache.solr.common.SolrException", "root-error-class","org.apache.solr.common.SolrException"], "msg":"4 out of 5 the property overlay to be of version 3 within 30 seconds! Failed cores: [https://127.0.0.1:56023/collection1_shard2_replica_n41/, https://127.0.0.1:54547/collection1_shard2_replica_n45/, https://127.0.0.1:52772/collection1_shard1_replica_n43/, https://127.0.0.1:50133/collection1_shard1_replica_n47/]", "trace":"org.apache.solr.common.SolrException: 4 out of 5 the property overlay to be of version 3 within 30 seconds! Failed cores: [https://127.0.0.1:56023/collection1_shard2_replica_n41/, https://127.0.0.1:54547/collection1_shard2_replica_n45/, https://127.0.0.1:52772/collection1_shard1_replica_n43/, https://127.0.0.1:50133/collection1_shard1_replica_n47/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:772)\n\tat org.apache.solr.handler.SolrConfigHandler.access$400(SolrConfigHandler.java:102)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:503)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:346)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.access$100(SolrConfigHandler.java:159)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:137)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2484)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:710)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:516)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:534)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)\n\tat java.lang.Thread.run(Thread.java:748)\n", "code":500}}
Stack Trace:
java.lang.AssertionError: {
"responseHeader":{
"status":500,
"QTime":32476},
"errorMessages":["4 out of 5 the property overlay to be of version 3 within 30 seconds! Failed cores: [https://127.0.0.1:56023/collection1_shard2_replica_n41/, https://127.0.0.1:54547/collection1_shard2_replica_n45/, https://127.0.0.1:52772/collection1_shard1_replica_n43/, https://127.0.0.1:50133/collection1_shard1_replica_n47/]\n"],
"WARNING":"This response format is experimental. It is likely to change in the future.",
"error":{
"metadata":[
"error-class","org.apache.solr.common.SolrException",
"root-error-class","org.apache.solr.common.SolrException"],
"msg":"4 out of 5 the property overlay to be of version 3 within 30 seconds! Failed cores: [https://127.0.0.1:56023/collection1_shard2_replica_n41/, https://127.0.0.1:54547/collection1_shard2_replica_n45/, https://127.0.0.1:52772/collection1_shard1_replica_n43/, https://127.0.0.1:50133/collection1_shard1_replica_n47/]",
"trace":"org.apache.solr.common.SolrException: 4 out of 5 the property overlay to be of version 3 within 30 seconds! Failed cores: [https://127.0.0.1:56023/collection1_shard2_replica_n41/, https://127.0.0.1:54547/collection1_shard2_replica_n45/, https://127.0.0.1:52772/collection1_shard1_replica_n43/, https://127.0.0.1:50133/collection1_shard1_replica_n47/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:772)\n\tat org.apache.solr.handler.SolrConfigHandler.access$400(SolrConfigHandler.java:102)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:503)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:346)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.access$100(SolrConfigHandler.java:159)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:137)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2484)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:710)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:516)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:534)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)\n\tat java.lang.Thread.run(Thread.java:748)\n",
"code":500}}
at __randomizedtesting.SeedInfo.seed([2BF42D8AD9FD7CA4:A3A012507701115C]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.core.TestSolrConfigHandler.runConfigCommand(TestSolrConfigHandler.java:185)
at org.apache.solr.core.TestSolrConfigHandler.reqhandlertests(TestSolrConfigHandler.java:253)
at org.apache.solr.handler.TestSolrConfigHandlerCloud.testReqHandlerAPIs(TestSolrConfigHandlerCloud.java:97)
at org.apache.solr.handler.TestSolrConfigHandlerCloud.test(TestSolrConfigHandlerCloud.java:67)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.update.SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeRapidAdds
Error Message:
softCommit: did not find a single commit
Stack Trace:
java.lang.AssertionError: softCommit: did not find a single commit
at __randomizedtesting.SeedInfo.seed([2BF42D8AD9FD7CA4:BE5A8BDD3BEF0ED9]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNotNull(Assert.java:526)
at org.apache.solr.update.SoftAutoCommitTest.assertRapidMultiCommitQueues(SoftAutoCommitTest.java:494)
at org.apache.solr.update.SoftAutoCommitTest.doTestSoftAndHardCommitMaxTimeRapidAdds(SoftAutoCommitTest.java:459)
at org.apache.solr.update.SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeRapidAdds(SoftAutoCommitTest.java:411)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.client.solrj.impl.CloudSolrClientTest.preferLocalShardsTest
Error Message:
Could not load collection from ZK: localShardsTestColl
Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: localShardsTestColl
at __randomizedtesting.SeedInfo.seed([D5E1F801E0D45786:292C60B81E03C07C]:0)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1188)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:832)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.client.solrj.impl.CloudSolrClientTest.queryWithPreferLocalShards(CloudSolrClientTest.java:456)
at org.apache.solr.client.solrj.impl.CloudSolrClientTest.preferLocalShardsTest(CloudSolrClientTest.java:437)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:110)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/localShardsTestColl/state.json
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
... 48 more
Build Log:
[...truncated 11766 lines...]
[junit4] Suite: org.apache.solr.cloud.DeleteReplicaTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/init-core-data-001
[junit4] 2> 733275 WARN (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=643 numCloses=643
[junit4] 2> 733275 INFO (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 733277 INFO (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 733292 INFO (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 733293 INFO (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001
[junit4] 2> 733293 INFO (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 733301 INFO (Thread-192) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 733302 INFO (Thread-192) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 733432 INFO (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:36541
[junit4] 2> 733463 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
[junit4] 2> 733472 ERROR (Thread-192) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 735213 INFO (jetty-launcher-154-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 735214 INFO (jetty-launcher-154-thread-3) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 735220 INFO (jetty-launcher-154-thread-4) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 735224 INFO (jetty-launcher-154-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@d87a547{/solr,null,AVAILABLE}
[junit4] 2> 735225 INFO (jetty-launcher-154-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5cf2639a{/solr,null,AVAILABLE}
[junit4] 2> 735243 INFO (jetty-launcher-154-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@8ce55ac{/solr,null,AVAILABLE}
[junit4] 2> 735302 INFO (jetty-launcher-154-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 735332 INFO (jetty-launcher-154-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@167b9c6b{SSL,[ssl, http/1.1]}{127.0.0.1:50213}
[junit4] 2> 735332 INFO (jetty-launcher-154-thread-3) [ ] o.e.j.s.Server Started @760069ms
[junit4] 2> 735332 INFO (jetty-launcher-154-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50213}
[junit4] 2> 735355 INFO (jetty-launcher-154-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@23fd37e7{SSL,[ssl, http/1.1]}{127.0.0.1:58341}
[junit4] 2> 735355 INFO (jetty-launcher-154-thread-4) [ ] o.e.j.s.Server Started @760092ms
[junit4] 2> 735355 INFO (jetty-launcher-154-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58341}
[junit4] 2> 735356 ERROR (jetty-launcher-154-thread-4) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-25T23:49:57.055Z
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1a1abeb8{SSL,[ssl, http/1.1]}{127.0.0.1:50473}
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-2) [ ] o.e.j.s.Server Started @760093ms
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50473}
[junit4] 2> 735356 ERROR (jetty-launcher-154-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 735357 INFO (jetty-launcher-154-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 735357 INFO (jetty-launcher-154-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-25T23:49:57.056Z
[junit4] 2> 735375 ERROR (jetty-launcher-154-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 735375 INFO (jetty-launcher-154-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 735375 INFO (jetty-launcher-154-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 735375 INFO (jetty-launcher-154-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 735375 INFO (jetty-launcher-154-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-25T23:49:57.074Z
[junit4] 2> 735376 INFO (jetty-launcher-154-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@a601c96{/solr,null,AVAILABLE}
[junit4] 2> 735455 INFO (jetty-launcher-154-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6b090788{SSL,[ssl, http/1.1]}{127.0.0.1:55321}
[junit4] 2> 735455 INFO (jetty-launcher-154-thread-1) [ ] o.e.j.s.Server Started @760192ms
[junit4] 2> 735455 INFO (jetty-launcher-154-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=55321}
[junit4] 2> 735455 ERROR (jetty-launcher-154-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 735455 INFO (jetty-launcher-154-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 735455 INFO (jetty-launcher-154-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 735455 INFO (jetty-launcher-154-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 735455 INFO (jetty-launcher-154-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-25T23:49:57.154Z
[junit4] 2> 735553 INFO (jetty-launcher-154-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 735555 INFO (jetty-launcher-154-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 735584 INFO (jetty-launcher-154-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 735591 INFO (jetty-launcher-154-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 735594 INFO (jetty-launcher-154-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36541/solr
[junit4] 2> 735715 INFO (jetty-launcher-154-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36541/solr
[junit4] 2> 735751 INFO (jetty-launcher-154-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36541/solr
[junit4] 2> 735819 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f55f1415a0005, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 736091 INFO (jetty-launcher-154-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36541/solr
[junit4] 2> 736227 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f55f1415a0007, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 736491 INFO (jetty-launcher-154-thread-2) [n:127.0.0.1:50473_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 736492 INFO (jetty-launcher-154-thread-2) [n:127.0.0.1:50473_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:50473_solr
[junit4] 2> 736499 INFO (jetty-launcher-154-thread-2) [n:127.0.0.1:50473_solr ] o.a.s.c.Overseer Overseer (id=98892211497336840-127.0.0.1:50473_solr-n_0000000000) starting
[junit4] 2> 736539 INFO (jetty-launcher-154-thread-2) [n:127.0.0.1:50473_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50473_solr
[junit4] 2> 736552 INFO (jetty-launcher-154-thread-3) [n:127.0.0.1:50213_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 736552 INFO (jetty-launcher-154-thread-3) [n:127.0.0.1:50213_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50213_solr
[junit4] 2> 736691 INFO (zkCallback-174-thread-1-processing-n:127.0.0.1:50213_solr) [n:127.0.0.1:50213_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 736701 INFO (OverseerStateUpdate-98892211497336840-127.0.0.1:50473_solr-n_0000000000) [n:127.0.0.1:50473_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 736994 INFO (zkCallback-177-thread-1-processing-n:127.0.0.1:58341_solr) [n:127.0.0.1:58341_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 736997 INFO (jetty-launcher-154-thread-4) [n:127.0.0.1:58341_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 737001 INFO (jetty-launcher-154-thread-4) [n:127.0.0.1:58341_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58341_solr
[junit4] 2> 737003 INFO (zkCallback-174-thread-1-processing-n:127.0.0.1:50213_solr) [n:127.0.0.1:50213_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 737007 INFO (zkCallback-177-thread-1-processing-n:127.0.0.1:58341_solr) [n:127.0.0.1:58341_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 737012 INFO (zkCallback-172-thread-1-processing-n:127.0.0.1:50473_solr) [n:127.0.0.1:50473_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 738052 INFO (jetty-launcher-154-thread-3) [n:127.0.0.1:50213_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50213.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 738143 INFO (jetty-launcher-154-thread-3) [n:127.0.0.1:50213_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50213.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 738143 INFO (jetty-launcher-154-thread-3) [n:127.0.0.1:50213_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50213.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 738179 INFO (jetty-launcher-154-thread-3) [n:127.0.0.1:50213_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001/node3/.
[junit4] 2> 739631 INFO (jetty-launcher-154-thread-1) [n:127.0.0.1:55321_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 739632 INFO (jetty-launcher-154-thread-1) [n:127.0.0.1:55321_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 739643 INFO (jetty-launcher-154-thread-1) [n:127.0.0.1:55321_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55321_solr
[junit4] 2> 739644 INFO (zkCallback-172-thread-1-processing-n:127.0.0.1:50473_solr) [n:127.0.0.1:50473_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 739645 INFO (zkCallback-174-thread-1-processing-n:127.0.0.1:50213_solr) [n:127.0.0.1:50213_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 739651 INFO (zkCallback-177-thread-1-processing-n:127.0.0.1:58341_solr) [n:127.0.0.1:58341_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 739715 INFO (zkCallback-176-thread-1-processing-n:127.0.0.1:55321_solr) [n:127.0.0.1:55321_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 743877 INFO (jetty-launcher-154-thread-4) [n:127.0.0.1:58341_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58341.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744111 INFO (jetty-launcher-154-thread-4) [n:127.0.0.1:58341_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58341.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744119 INFO (jetty-launcher-154-thread-4) [n:127.0.0.1:58341_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58341.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744120 INFO (jetty-launcher-154-thread-4) [n:127.0.0.1:58341_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001/node4/.
[junit4] 2> 744247 INFO (jetty-launcher-154-thread-2) [n:127.0.0.1:50473_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50473.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744278 INFO (jetty-launcher-154-thread-2) [n:127.0.0.1:50473_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50473.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744279 INFO (jetty-launcher-154-thread-2) [n:127.0.0.1:50473_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50473.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744328 INFO (jetty-launcher-154-thread-2) [n:127.0.0.1:50473_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001/node2/.
[junit4] 2> 744518 INFO (jetty-launcher-154-thread-1) [n:127.0.0.1:55321_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55321.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744599 INFO (jetty-launcher-154-thread-1) [n:127.0.0.1:55321_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55321.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744600 INFO (jetty-launcher-154-thread-1) [n:127.0.0.1:55321_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55321.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744601 INFO (jetty-launcher-154-thread-1) [n:127.0.0.1:55321_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001/node1/.
[junit4] 2> 745763 INFO (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 745764 INFO (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36541/solr ready
[junit4] 2> 746042 INFO (TEST-DeleteReplicaTest.deleteReplicaAndVerifyDirectoryCleanup-seed#[2BF42D8AD9FD7CA4]) [ ] o.a.s.SolrTestCaseJ4 ###Starting deleteReplicaAndVerifyDirectoryCleanup
[junit4] 2> 746449 INFO (qtp710352259-1234) [n:127.0.0.1:50213_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&name=deletereplica_test&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 746531 INFO (OverseerThreadFactory-432-thread-1-processing-n:127.0.0.1:50473_solr) [n:127.0.0.1:50473_solr ] o.a.s.c.CreateCollectionCmd Create collection deletereplica_test
[junit4] 2> 746682 INFO (OverseerStateUpdate-98892211497336840-127.0.0.1:50473_solr-n_0000000000) [n:127.0.0.1:50473_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"deletereplica_test",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"deletereplica_test_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:58341/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 746713 INFO (OverseerStateUpdate-98892211497336840-127.0.0.1:50473_solr-n_0000000000) [n:127.0.0.1:50473_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"deletereplica_test",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"deletereplica_test_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:50213/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 749121 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=deletereplica_test_shard1_replica_n2&action=CREATE&numShards=1&collection=deletereplica_test&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 749122 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 749173 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 758284 WARN (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker-SendThread(127.0.0.1:36541)) [ ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 11735ms for sessionid 0x15f55f1415a000e
[junit4] 2> 758421 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=deletereplica_test_shard1_replica_n1&action=CREATE&numShards=1&collection=deletereplica_test&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 758423 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 758429 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.s.IndexSchema [deletereplica_test_shard1_replica_n2] Schema name=minimal
[junit4] 2> 758636 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 758636 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'deletereplica_test_shard1_replica_n2' using configuration from collection deletereplica_test, trusted=true
[junit4] 2> 758637 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50213.solr.core.deletereplica_test.shard1.replica_n2' (registry 'solr.core.deletereplica_test.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 758642 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 758642 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.c.SolrCore [[deletereplica_test_shard1_replica_n2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001/node3/deletereplica_test_shard1_replica_n2], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001/node3/./deletereplica_test_shard1_replica_n2/data/]
[junit4] 2> 758643 WARN (zkCallback-185-thread-2) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5b88fda5 name: ZooKeeperConnection Watcher:127.0.0.1:36541/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 758643 WARN (zkCallback-185-thread-2) [ ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 758648 INFO (zkCallback-174-thread-1-processing-n:127.0.0.1:50213_solr) [n:127.0.0.1:50213_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deletereplica_test/state.json] for collection [deletereplica_test] has occurred - updating... (live nodes size: [4])
[junit4] 2> 758707 INFO (zkCallback-177-thread-1-processing-n:127.0.0.1:58341_solr) [n:127.0.0.1:58341_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deletereplica_test/state.json] for collection [deletereplica_test] has occurred - updating... (live nodes size: [4])
[junit4] 2> 758707 INFO (zkCallback-177-thread-2-processing-n:127.0.0.1:58341_solr) [n:127.0.0.1:58341_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deletereplica_test/state.json] for collection [deletereplica_test] has occurred - updating... (live nodes size: [4])
[junit4] 2> 759403 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 759403 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 759434 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 759434 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 759633 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@38c628b4[deletereplica_test_shard1_replica_n2] main]
[junit4] 2> 759635 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 759636 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 759637 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 759876 INFO (searcherExecutor-441-thread-1-processing-n:127.0.0.1:50213_solr x:deletereplica_test_shard1_replica_n2 s:shard1 c:deletereplica_test r:core_node4) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.c.SolrCore [deletereplica_test_shard1_replica_n2] Registered new searcher Searcher@38c628b4[deletereplica_test_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 759899 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr c:deletereplica_test s:shard1 r:core_node3 x:deletereplica_test_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 759935 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582275411715293184
[junit4] 2> 759956 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 759995 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr c:deletereplica_test s:shard1 r:core_node3 x:deletereplica_test_shard1_replica_n1] o.a.s.s.IndexSchema [deletereplica_test_shard1_replica_n1] Schema name=minimal
[junit4] 2> 759997 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr c:deletereplica_test s:shard1 r:core_node3 x:deletereplica_test_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 759997 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr c:deletereplica_test s:shard1 r:core_node3 x:deletereplica_test_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'deletereplica_test_shard1_replica_n1' using configuration from collection deletereplica_test, trusted=true
[junit4] 2> 759998 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr c:deletereplica_test s:shard1 r:core_node3 x:deletereplica_test_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58341.solr.core.deletereplica_test.shard1.replica_n1' (registry 'solr.core.deletereplica_test.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 759998 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr c:deletereplica_test s:shard1 r:core_node3 x:deletereplica_test_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 759998 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr c:deletereplica_test s:shard1 r:core_node3 x:deletereplica_test_shard1_replica_n1] o.a.s.c.SolrCore [[deletereplica_test_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001/node4/deletereplica_test_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001/node4/./deletereplica_test_shard1_replica_n1/data/]
[junit4] 2> 760059 INFO (zkCallback-174-thread-1-processing-n:127.0.0.1:50213_solr) [n:127.0.0.1:50213_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deletereplica_test/state.json] for collection [deletereplica_test] has occurred - updating... (live nodes size: [4])
[junit4] 2> 760059 INFO (zkCallback-177-thread-2-processing-n:127.0.0.1:58341_solr) [n:127.0.0.1:58341_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deletereplica_test/state.json] for collection [deletereplica_test] has occurred - updating... (live nodes size: [4])
[junit4] 2> 760059 INFO (zkCallback-177-thread-1-processing-n:127.0.0.1:58341_solr) [n:127.0.0.1:58341_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deletereplica_test/state.json] for collection [deletereplica_test] has occurred - updating... (live nodes size: [4])
[junit4] 2> 760287 WARN (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker-SendThread(127.0.0.1:36541)) [ ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15f55f1415a000e has expired
[junit4] 2> 760323 WARN (zkCallback-185-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5b88fda5 name: ZooKeeperConnection Watcher:127.0.0.1:36541/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
[junit4] 2> 760323 WARN (zkCallback-185-thread-1) [ ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
[junit4] 2> 760323 WARN (zkCallback-185-thread-1) [ ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
[junit4] 2> 760439 INFO (zkCallback-185-thread-1) [ ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
[junit4] 2> 760656 INFO (zkCallback-185-thread-1) [ ] o.a.s.c.c.DefaultConnectionStrategy Reconnected to ZooKeeper
[ju
[...truncated too long message...]
re.2nd_collection.shard1.replica_n1, tag=1848148207
[junit4] 2> 463140 INFO (coreCloseExecutor-301-thread-6) [n:127.0.0.1:39390_solr c:2nd_collection s:shard1 r:core_node3 x:2nd_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@570c1239: rootName = solr_39390, domain = solr.core.2nd_collection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.2nd_collection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@3c5e08cb
[junit4] 2> 463147 INFO (coreCloseExecutor-303-thread-3) [n:127.0.0.1:59769_solr c:localShardsTestColl s:shard1 r:core_node2 x:localShardsTestColl_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.localShardsTestColl.shard1.leader, tag=823262030
[junit4] 2> 463172 INFO (coreCloseExecutor-301-thread-7) [n:127.0.0.1:39390_solr c:multicollection1 s:shard2 r:core_node4 x:multicollection1_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.multicollection1.shard2.replica_n2, tag=1956061779
[junit4] 2> 463199 INFO (coreCloseExecutor-301-thread-7) [n:127.0.0.1:39390_solr c:multicollection1 s:shard2 r:core_node4 x:multicollection1_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@24acb18a: rootName = solr_39390, domain = solr.core.multicollection1.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.multicollection1.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@1f6619b8
[junit4] 2> 463344 INFO (coreCloseExecutor-301-thread-1) [n:127.0.0.1:39390_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard1.leader, tag=1666671325
[junit4] 2> 463345 INFO (coreCloseExecutor-301-thread-6) [n:127.0.0.1:39390_solr c:2nd_collection s:shard1 r:core_node3 x:2nd_collection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.2nd_collection.shard1.leader, tag=1848148207
[junit4] 2> 463345 INFO (coreCloseExecutor-301-thread-5) [n:127.0.0.1:39390_solr c:localShardsTestColl s:shard3 r:core_node18 x:localShardsTestColl_shard3_replica_n17] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.localShardsTestColl.shard3.leader, tag=1216561606
[junit4] 2> 463345 INFO (coreCloseExecutor-301-thread-7) [n:127.0.0.1:39390_solr c:multicollection1 s:shard2 r:core_node4 x:multicollection1_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.multicollection1.shard2.leader, tag=1956061779
[junit4] 2> 463347 INFO (coreCloseExecutor-301-thread-4) [n:127.0.0.1:39390_solr c:localShardsTestColl s:shard1 r:core_node6 x:localShardsTestColl_shard1_replica_n5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.localShardsTestColl.shard1.leader, tag=1638113748
[junit4] 2> 463367 INFO (coreCloseExecutor-302-thread-4) [n:127.0.0.1:60025_solr c:localShardsTestColl s:shard2 r:core_node10 x:localShardsTestColl_shard2_replica_n9] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.localShardsTestColl.shard2.leader, tag=1686851154
[junit4] 2> 463367 INFO (coreCloseExecutor-302-thread-5) [n:127.0.0.1:60025_solr c:2nd_collection s:shard2 r:core_node4 x:2nd_collection_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.2nd_collection.shard2.replica_n2, tag=548992199
[junit4] 2> 463367 INFO (coreCloseExecutor-302-thread-5) [n:127.0.0.1:60025_solr c:2nd_collection s:shard2 r:core_node4 x:2nd_collection_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@807e8be: rootName = solr_60025, domain = solr.core.2nd_collection.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.2nd_collection.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@503125f7
[junit4] 2> 463383 INFO (coreCloseExecutor-302-thread-5) [n:127.0.0.1:60025_solr c:2nd_collection s:shard2 r:core_node4 x:2nd_collection_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.2nd_collection.shard2.leader, tag=548992199
[junit4] 2> 463595 INFO (coreCloseExecutor-302-thread-6) [n:127.0.0.1:60025_solr c:multicollection2 s:shard2 r:core_node4 x:multicollection2_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.multicollection2.shard2.replica_n2, tag=1028853355
[junit4] 2> 463595 INFO (coreCloseExecutor-302-thread-6) [n:127.0.0.1:60025_solr c:multicollection2 s:shard2 r:core_node4 x:multicollection2_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4b88932e: rootName = solr_60025, domain = solr.core.multicollection2.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.multicollection2.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@357c1301
[junit4] 2> 463635 INFO (coreCloseExecutor-302-thread-6) [n:127.0.0.1:60025_solr c:multicollection2 s:shard2 r:core_node4 x:multicollection2_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.multicollection2.shard2.leader, tag=1028853355
[junit4] 2> 464586 INFO (jetty-closer-97-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2a3505a1{/solr,null,UNAVAILABLE}
[junit4] 2> 464729 INFO (jetty-closer-97-thread-1) [ ] o.a.s.c.Overseer Overseer (id=98892935048396807-127.0.0.1:59769_solr-n_0000000000) closing
[junit4] 2> 464729 INFO (OverseerStateUpdate-98892935048396807-127.0.0.1:59769_solr-n_0000000000) [n:127.0.0.1:59769_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:59769_solr
[junit4] 2> 464990 WARN (OverseerAutoScalingTriggerThread-98892935048396807-127.0.0.1:59769_solr-n_0000000000) [n:127.0.0.1:59769_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 465247 INFO (jetty-closer-97-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@22af484b{/solr,null,UNAVAILABLE}
[junit4] 2> 465947 INFO (jetty-closer-97-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@47712441{/solr,null,UNAVAILABLE}
[junit4] 2> 465949 ERROR (SUITE-CloudSolrClientTest-seed#[D5E1F801E0D45786]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 465949 INFO (SUITE-CloudSolrClientTest-seed#[D5E1F801E0D45786]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:54220 54220
[junit4] 2> 466063 INFO (Thread-82) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:54220 54220
[junit4] 2> 466083 WARN (Thread-82) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/aliases.json
[junit4] 2> 3 /solr/security.json
[junit4] 2> 3 /solr/configs/conf
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 44 /solr/collections/localShardsTestColl/state.json
[junit4] 2> 33 /solr/collections/foo/state.json
[junit4] 2> 17 /solr/collections/collection1/state.json
[junit4] 2> 16 /solr/collections/2nd_collection/state.json
[junit4] 2> 16 /solr/collections/multicollection1/state.json
[junit4] 2> 15 /solr/collections/multicollection2/state.json
[junit4] 2> 14 /solr/collections/nemesis/state.json
[junit4] 2> 8 /solr/clusterprops.json
[junit4] 2> 8 /solr/clusterstate.json
[junit4] 2> 6 /solr/collections/overwrite/state.json
[junit4] 2> 2 /solr/collections/localShardsTestColl/leader_elect/shard3/election/98892935048396808-core_node16-n_0000000000
[junit4] 2> 2 /solr/collections/foo/leader_elect/shard1/election/98892935048396808-core_node4-n_0000000000
[junit4] 2> 2 /solr/collections/localShardsTestColl/leader_elect/shard1/election/98892935048396807-core_node2-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/live_nodes
[junit4] 2> 8 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.impl.CloudSolrClientTest_D5E1F801E0D45786-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {multiDefault=PostingsFormat(name=MockRandom), a_t=PostingsFormat(name=MockRandom), title_s=PostingsFormat(name=MockRandom), id=FST50, text=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{_version_=DocValuesFormat(name=Direct), multiDefault=DocValuesFormat(name=Lucene70), title_s=DocValuesFormat(name=Lucene70), intDefault=DocValuesFormat(name=Direct), id=DocValuesFormat(name=Asserting), timestamp=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=981, maxMBSortInHeap=6.251189272312671, sim=RandomSimilarity(queryNorm=true): {}, locale=ga-IE, timezone=Etc/GMT-8
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=147777368,total=442499072
[junit4] 2> NOTE: All tests run in this JVM: [GreaterThanEqualToEvaluatorTest, SolrExampleEmbeddedTest, UuidEvaluatorTest, SolrExampleStreamingBinaryTest, TestCollectionStateWatchers, CloudSolrClientTest]
[junit4] Completed [53/145 (1!)] on J1 in 142.96s, 15 tests, 1 error <<< FAILURES!
[...truncated 46589 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 192 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/192/
4 tests failed.
FAILED: org.apache.solr.cloud.TestTlogReplica.testKillTlogReplica
Error Message:
Error from server at https://127.0.0.1:43641/solr/tlog_replica_test_kill_tlog_replica_shard1_replica_t2: ClusterState says we are the leader (https://127.0.0.1:43641/solr/tlog_replica_test_kill_tlog_replica_shard1_replica_t2), but locally we don't think so. Request came from null
Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:43641/solr/tlog_replica_test_kill_tlog_replica_shard1_replica_t2: ClusterState says we are the leader (https://127.0.0.1:43641/solr/tlog_replica_test_kill_tlog_replica_shard1_replica_t2), but locally we don't think so. Request came from null
at __randomizedtesting.SeedInfo.seed([E99EB240D3F155EC:36AC76852252DAE1]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:541)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:998)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:927)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
at org.apache.solr.cloud.TestTlogReplica.testKillTlogReplica(TestTlogReplica.java:445)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:43641/solr/tlog_replica_test_kill_tlog_replica_shard1_replica_t2: ClusterState says we are the leader (https://127.0.0.1:43641/solr/tlog_replica_test_kill_tlog_replica_shard1_replica_t2), but locally we don't think so. Request came from null
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:518)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
FAILED: org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test
Error Message:
Expected 2 of 3 replicas to be active but only found 1; [core_node5:{"core":"c8n_1x3_lf_shard1_replica_t2","base_url":"http://127.0.0.1:45754/pjt","node_name":"127.0.0.1:45754_pjt","state":"active","type":"TLOG","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/17)={ "pullReplicas":"0", "replicationFactor":"1", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{ "core_node4":{ "state":"down", "base_url":"http://127.0.0.1:42503/pjt", "core":"c8n_1x3_lf_shard1_replica_t1", "node_name":"127.0.0.1:42503_pjt", "type":"TLOG"}, "core_node5":{ "core":"c8n_1x3_lf_shard1_replica_t2", "base_url":"http://127.0.0.1:45754/pjt", "node_name":"127.0.0.1:45754_pjt", "state":"active", "type":"TLOG", "leader":"true"}, "core_node6":{ "core":"c8n_1x3_lf_shard1_replica_t3", "base_url":"http://127.0.0.1:40853/pjt", "node_name":"127.0.0.1:40853_pjt", "state":"down", "type":"TLOG"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"0", "tlogReplicas":"3"}
Stack Trace:
java.lang.AssertionError: Expected 2 of 3 replicas to be active but only found 1; [core_node5:{"core":"c8n_1x3_lf_shard1_replica_t2","base_url":"http://127.0.0.1:45754/pjt","node_name":"127.0.0.1:45754_pjt","state":"active","type":"TLOG","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/17)={
"pullReplicas":"0",
"replicationFactor":"1",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{
"core_node4":{
"state":"down",
"base_url":"http://127.0.0.1:42503/pjt",
"core":"c8n_1x3_lf_shard1_replica_t1",
"node_name":"127.0.0.1:42503_pjt",
"type":"TLOG"},
"core_node5":{
"core":"c8n_1x3_lf_shard1_replica_t2",
"base_url":"http://127.0.0.1:45754/pjt",
"node_name":"127.0.0.1:45754_pjt",
"state":"active",
"type":"TLOG",
"leader":"true"},
"core_node6":{
"core":"c8n_1x3_lf_shard1_replica_t3",
"base_url":"http://127.0.0.1:40853/pjt",
"node_name":"127.0.0.1:40853_pjt",
"state":"down",
"type":"TLOG"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"0",
"tlogReplicas":"3"}
at __randomizedtesting.SeedInfo.seed([E99EB240D3F155EC:61CA8D9A7D0D3814]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:169)
at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:56)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestPrepRecovery.testLeaderUnloaded
Error Message:
Expected collection: testLeaderUnloaded to be live with 1 shard and 2 replicas null Live Nodes: [127.0.0.1:36908_solr, 127.0.0.1:41335_solr] Last available state: null
Stack Trace:
java.lang.AssertionError: Expected collection: testLeaderUnloaded to be live with 1 shard and 2 replicas
null
Live Nodes: [127.0.0.1:36908_solr, 127.0.0.1:41335_solr]
Last available state: null
at __randomizedtesting.SeedInfo.seed([E99EB240D3F155EC:BA39A1B62EDDBBEE]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
at org.apache.solr.cloud.TestPrepRecovery.testLeaderUnloaded(TestPrepRecovery.java:59)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestPrepRecovery.testLeaderNotResponding
Error Message:
Expected collection: testLeaderNotResponding to be live with 1 shard and 1 replicas null Live Nodes: [127.0.0.1:36908_solr, 127.0.0.1:41335_solr] Last available state: null
Stack Trace:
java.lang.AssertionError: Expected collection: testLeaderNotResponding to be live with 1 shard and 1 replicas
null
Live Nodes: [127.0.0.1:36908_solr, 127.0.0.1:41335_solr]
Last available state: null
at __randomizedtesting.SeedInfo.seed([E99EB240D3F155EC:B0C29A2794869553]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
at org.apache.solr.cloud.TestPrepRecovery.testLeaderNotResponding(TestPrepRecovery.java:93)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13440 lines...]
[junit4] Suite: org.apache.solr.cloud.LeaderFailoverAfterPartitionTest
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/init-core-data-001
[junit4] 2> 4792246 WARN (SUITE-LeaderFailoverAfterPartitionTest-seed#[E99EB240D3F155EC]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=7 numCloses=7
[junit4] 2> 4792246 INFO (SUITE-LeaderFailoverAfterPartitionTest-seed#[E99EB240D3F155EC]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 4792276 INFO (SUITE-LeaderFailoverAfterPartitionTest-seed#[E99EB240D3F155EC]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 4792276 INFO (SUITE-LeaderFailoverAfterPartitionTest-seed#[E99EB240D3F155EC]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 4792276 INFO (SUITE-LeaderFailoverAfterPartitionTest-seed#[E99EB240D3F155EC]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /pjt/
[junit4] 2> 4792277 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 4792313 INFO (Thread-3079) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 4792313 INFO (Thread-3079) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 4792357 ERROR (Thread-3079) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 4792406 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.ZkTestServer start zk server on port:46280
[junit4] 2> 4792582 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 4792584 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 4792617 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 4792618 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 4792619 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 4792619 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 4792620 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 4792620 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 4792639 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 4792640 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 4792640 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 4792654 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
[junit4] 2> 4793291 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 4793365 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1f125ff{/pjt,null,AVAILABLE}
[junit4] 2> 4793365 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@361ed473{HTTP/1.1,[http/1.1]}{127.0.0.1:44291}
[junit4] 2> 4793365 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.Server Started @4806049ms
[junit4] 2> 4793365 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/tempDir-001/control/data, replicaType=NRT, hostContext=/pjt, hostPort=42503, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/control-001/cores}
[junit4] 2> 4793365 ERROR (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4793366 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 4793366 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4793366 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 4793366 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-24T12:09:46.134Z
[junit4] 2> 4793401 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 4793401 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/control-001/solr.xml
[junit4] 2> 4793404 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 4793404 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 4793418 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 4793454 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46280/solr
[junit4] 2> 4795489 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:42503_pjt ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 4795489 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:42503_pjt ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:42503_pjt
[junit4] 2> 4795490 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:42503_pjt ] o.a.s.c.Overseer Overseer (id=98883796015644676-127.0.0.1:42503_pjt-n_0000000000) starting
[junit4] 2> 4795517 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:42503_pjt ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42503_pjt
[junit4] 2> 4795567 INFO (OverseerStateUpdate-98883796015644676-127.0.0.1:42503_pjt-n_0000000000) [n:127.0.0.1:42503_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4795772 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:42503_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4795829 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:42503_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4795829 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:42503_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4795843 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:42503_pjt ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/control-001/cores
[junit4] 2> 4795929 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4795930 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46280/solr ready
[junit4] 2> 4795931 INFO (SocketProxy-Acceptor-42503) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=42696,localport=42503], receiveBufferSize:531000
[junit4] 2> 4795931 INFO (SocketProxy-Acceptor-42503) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=44291,localport=39882], receiveBufferSize=530904
[junit4] 2> 4795945 INFO (qtp1614536027-20976) [n:127.0.0.1:42503_pjt ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:42503_pjt&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4795950 INFO (OverseerThreadFactory-6955-thread-1-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 4796115 INFO (SocketProxy-Acceptor-42503) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=42702,localport=42503], receiveBufferSize:531000
[junit4] 2> 4796116 INFO (SocketProxy-Acceptor-42503) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=44291,localport=39888], receiveBufferSize=530904
[junit4] 2> 4796134 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 4796134 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 4796257 INFO (zkCallback-5067-thread-1-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4797224 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 4797278 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 4798132 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 4798189 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 4798190 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4798190 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 4798190 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 4798208 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=26, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 4798224 WARN (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 4798518 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 4798518 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 4798519 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 4798519 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 4798520 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=36.0888671875, floorSegmentMB=0.9384765625, forceMergeDeletesPctAllowed=2.6420731854266144, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 4798520 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@93a6a7[control_collection_shard1_replica_n1] main]
[junit4] 2> 4798534 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 4798534 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 4798534 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 4798535 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582140742752534528
[junit4] 2> 4798537 INFO (searcherExecutor-6958-thread-1-processing-n:127.0.0.1:42503_pjt x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@93a6a7[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 4798572 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 4798590 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 4798590 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:42503/pjt/control_collection_shard1_replica_n1/
[junit4] 2> 4798590 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 4798590 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:42503/pjt/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 4798590 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 4798592 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:42503/pjt/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 4798615 INFO (zkCallback-5067-thread-2-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4798643 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 4798644 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2527
[junit4] 2> 4798728 INFO (qtp1614536027-20976) [n:127.0.0.1:42503_pjt ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 4798818 INFO (zkCallback-5067-thread-2-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4799733 INFO (qtp1614536027-20976) [n:127.0.0.1:42503_pjt ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:42503_pjt&wt=javabin&version=2} status=0 QTime=3787
[junit4] 2> 4799782 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4799783 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46280/solr ready
[junit4] 2> 4799783 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 4799783 INFO (SocketProxy-Acceptor-42503) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=42722,localport=42503], receiveBufferSize:531000
[junit4] 2> 4799806 INFO (SocketProxy-Acceptor-42503) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=44291,localport=39908], receiveBufferSize=530904
[junit4] 2> 4799806 INFO (qtp1614536027-20973) [n:127.0.0.1:42503_pjt ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4799807 INFO (OverseerThreadFactory-6955-thread-2-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 4799807 INFO (OverseerCollectionConfigSetProcessor-98883796015644676-127.0.0.1:42503_pjt-n_0000000000) [n:127.0.0.1:42503_pjt ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 4799808 WARN (OverseerThreadFactory-6955-thread-2-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 4800054 INFO (qtp1614536027-20973) [n:127.0.0.1:42503_pjt ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 4800078 INFO (qtp1614536027-20973) [n:127.0.0.1:42503_pjt ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=272
[junit4] 2> 4800565 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-1-001 of type TLOG
[junit4] 2> 4800565 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 4800596 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6df48259{/pjt,null,AVAILABLE}
[junit4] 2> 4800597 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@53c235ce{HTTP/1.1,[http/1.1]}{127.0.0.1:35478}
[junit4] 2> 4800597 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.Server Started @4813280ms
[junit4] 2> 4800597 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/pjt, hostPort=45754, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-1-001/cores}
[junit4] 2> 4800597 ERROR (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4800597 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 4800597 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4800597 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 4800597 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-24T12:09:53.365Z
[junit4] 2> 4800599 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 4800599 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-1-001/solr.xml
[junit4] 2> 4800602 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 4800602 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 4800618 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 4800661 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46280/solr
[junit4] 2> 4800772 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4800787 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:45754_pjt ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 4800788 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:45754_pjt ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45754_pjt
[junit4] 2> 4800789 INFO (zkCallback-5075-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 4800789 INFO (zkCallback-5067-thread-2-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 4800825 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45754_pjt) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 4801202 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:45754_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4801240 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:45754_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4801240 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:45754_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4801242 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:45754_pjt ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-1-001/cores
[junit4] 2> 4801409 INFO (qtp1614536027-20979) [n:127.0.0.1:42503_pjt ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:45754_pjt&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4801445 INFO (OverseerCollectionConfigSetProcessor-98883796015644676-127.0.0.1:42503_pjt-n_0000000000) [n:127.0.0.1:42503_pjt ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 4801466 INFO (OverseerThreadFactory-6955-thread-3-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:45754_pjt for creating new replica
[junit4] 2> 4801478 INFO (SocketProxy-Acceptor-45754) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=43216,localport=45754], receiveBufferSize:531000
[junit4] 2> 4801490 INFO (SocketProxy-Acceptor-45754) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=35478,localport=48322], receiveBufferSize=530904
[junit4] 2> 4801495 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 4801495 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 4801661 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45754_pjt) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4802561 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 4802614 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema [collection1_shard2_replica_t41] Schema name=test
[junit4] 2> 4803363 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 4803440 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t41' using configuration from collection collection1, trusted=true
[junit4] 2> 4803441 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t41' (registry 'solr.core.collection1.shard2.replica_t41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4803441 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 4803441 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [[collection1_shard2_replica_t41] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-1-001/cores/collection1_shard2_replica_t41], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-1-001/cores/collection1_shard2_replica_t41/data/]
[junit4] 2> 4803456 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=26, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 4803472 WARN (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 4803767 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 4803767 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 4803768 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 4803768 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 4803769 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=36.0888671875, floorSegmentMB=0.9384765625, forceMergeDeletesPctAllowed=2.6420731854266144, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 4803770 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.SolrIndexSearcher Opening [Searcher@4b3bf8d4[collection1_shard2_replica_t41] main]
[junit4] 2> 4803787 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 4803788 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 4803788 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 4803788 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582140748260704256
[junit4] 2> 4803791 INFO (searcherExecutor-6969-thread-1-processing-n:127.0.0.1:45754_pjt x:collection1_shard2_replica_t41 s:shard2 c:collection1) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [collection1_shard2_replica_t41] Registered new searcher Searcher@4b3bf8d4[collection1_shard2_replica_t41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 4803805 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 4803805 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 4803805 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45754/pjt/collection1_shard2_replica_t41/
[junit4] 2> 4803805 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 4803805 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy http://127.0.0.1:45754/pjt/collection1_shard2_replica_t41/ has no replicas
[junit4] 2> 4803805 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 4803805 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController collection1_shard2_replica_t41 stopping background replication from leader
[junit4] 2> 4803807 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45754/pjt/collection1_shard2_replica_t41/ shard2
[junit4] 2> 4803844 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45754_pjt) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4803901 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 4803902 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2407
[junit4] 2> 4803954 INFO (qtp1614536027-20979) [n:127.0.0.1:42503_pjt ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:45754_pjt&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=2545
[junit4] 2> 4804020 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45754_pjt) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4804634 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-2-001 of type TLOG
[junit4] 2> 4804635 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 4804707 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@63f9eea6{/pjt,null,AVAILABLE}
[junit4] 2> 4804707 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@64b6284c{HTTP/1.1,[http/1.1]}{127.0.0.1:44448}
[junit4] 2> 4804707 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.Server Started @4817390ms
[junit4] 2> 4804707 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/pjt, hostPort=34304, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-2-001/cores}
[junit4] 2> 4804707 ERROR (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4804722 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 4804723 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4804723 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 4804723 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-24T12:09:57.491Z
[junit4] 2> 4804779 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 4804779 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-2-001/solr.xml
[junit4] 2> 4804783 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 4804783 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 4804921 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 4804991 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46280/solr
[junit4] 2> 4805068 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:34304_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 4805102 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:34304_pjt ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 4805103 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:34304_pjt ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34304_pjt
[junit4] 2> 4805104 INFO (zkCallback-5075-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 4805104 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45754_pjt) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 4805117 INFO (zkCallback-5067-thread-1-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 4805141 INFO (zkCallback-5086-thread-1-processing-n:127.0.0.1:34304_pjt) [n:127.0.0.1:34304_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 4805402 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:34304_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4805442 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:34304_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4805442 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:34304_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4805459 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:34304_pjt ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-2-001/cores
[junit4] 2> 4805491 INFO (OverseerCollectionConfigSetProcessor-98883796015644676-127.0.0.1:42503_pjt-n_0000000000) [n:127.0.0.1:42503_pjt ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 4805711 INFO (SocketProxy-Acceptor-34304) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=50844,localport=34304], receiveBufferSize:531000
[junit4] 2> 4805727 INFO (SocketProxy-Acceptor-34304) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=44448,localport=46960], receiveBufferSize=530904
[junit4] 2> 4805748 INFO (qtp1003193669-21062) [n:127.0.0.1:34304_pjt ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:34304_pjt&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4805797 INFO (OverseerThreadFactory-6955-thread-4-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:34304_pjt for creating new replica
[junit4] 2> 4805833 INFO (SocketProxy-Acceptor-34304) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=50848,localport=34304], receiveBufferSize:531000
[junit4] 2> 4805834 INFO (SocketProxy-Acceptor-34304) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=44448,localport=46968], receiveBufferSize=530904
[junit4] 2> 4805838 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 4805839 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 4805953 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45754_pjt) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4805967 INFO (zkCallback-5086-thread-1-processing-n:127.0.0.1:34304_pjt) [n:127.0.0.1:34304_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4806936 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 4806994 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema [collection1_shard1_replica_t43] Schema name=test
[junit4] 2> 4807724 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 4807783 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t43' using configuration from collection collection1, trusted=true
[junit4] 2> 4807797 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t43' (registry 'solr.core.collection1.shard1.replica_t43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4807797 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 4807797 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [[collection1_shard1_replica_t43] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-2-001/cores/collection1_shard1_replica_t43], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-2-001/cores/collection1_shard1_replica_t43/data/]
[junit4] 2> 4807853 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=26, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 4807871 WARN (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 4808034 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 4808034 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 4808035 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 4808035 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 4808053 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=36.0888671875, floorSegmentMB=0.9384765625, forceMergeDeletesPctAllowed=2.6420731854266144, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 4808053 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.SolrIndexSearcher Opening [Searcher@76149f2e[collection1_shard1_replica_t43] main]
[junit4] 2> 4808054 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 4808054 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 4808054 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 4808055 INFO (searcherExecutor-6980-thread-1-processing-n:127.0.0.1:34304_pjt x:collection1_shard1_replica_t43 s:shard1 c:collection1) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [collection1_shard1_replica_t43] Registered new searcher Searcher@76149f2e[collection1_shard1_replica_t43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 4808055 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582140752734978048
[junit4] 2> 4808071 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 4808071 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 4808071 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:34304/pjt/collection1_shard1_replica_t43/
[junit4] 2> 4808071 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 4808071 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy http://127.0.0.1:34304/pjt/collection1_shard1_replica_t43/ has no replicas
[junit4] 2> 4808071 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 4808072 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController collection1_shard1_replica_t43 stopping background replication from leader
[junit4] 2> 4808090 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:34304/pjt/collection1_shard1_replica_t43/ shard1
[junit4] 2> 4808092 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45754_pjt) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4808100 INFO (zkCallback-5086-thread-1-processing-n:127.0.0.1:34304_pjt) [n:127.0.0.1:34304_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4808140 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 4808174 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2335
[junit4] 2> 4808176 INFO (qtp1003193669-21062) [n:127.0.0.1:34304_pjt ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:34304_pjt&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=2427
[junit4] 2> 4808322 INFO (zkCallback-5086-thread-1-processing-n:127.0.0.1:34304_pjt) [n:127.0.0.1:34304_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4808323 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45754_pjt) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4808855 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-3-001 of type TLOG
[junit4] 2> 4808855 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 4808856 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3f402781{/pjt,null,AVAILABLE}
[junit4] 2> 4808869 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@72cfd6fe{HTTP/1.1,[http/1.1]}{127.0.0.1:39176}
[junit4] 2> 4808869 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.Server Started @4821552ms
[junit4] 2> 4808869 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/tempDir-001/jetty3, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/pjt, hostPort=40853, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-3-001/cores}
[junit4] 2> 4808869 ERROR (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4808870 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 4808870 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4808870 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 4808870 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-24T12:10:01.638Z
[junit4] 2> 4808938 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 4808938 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-3-001/solr.xml
[junit4] 2> 4808941 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 4808941 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 4808955 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 4808992 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46280/solr
[junit4] 2> 4809138 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:40853_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 4809139 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:40853_pjt ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 4809157 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:40853_pjt ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40853_pjt
[junit4] 2> 4809160 I
[...truncated too long message...]
003) closing
[junit4] 2> 5666190 INFO (zkCallback-4578-thread-3-processing-n:127.0.0.1:46588_solr) [n:127.0.0.1:46588_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 5666197 WARN (OverseerAutoScalingTriggerThread-98883842714501134-127.0.0.1:46588_solr-n_0000000003) [n:127.0.0.1:46588_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 5666205 INFO (OverseerStateUpdate-98883842714501134-127.0.0.1:46588_solr-n_0000000003) [n:127.0.0.1:46588_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:46588_solr
[junit4] 2> 5666217 INFO (jetty-closer-4545-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@17049a9f{/solr,null,UNAVAILABLE}
[junit4] 2> 5666233 ERROR (SUITE-TestTlogReplica-seed#[E99EB240D3F155EC]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 5666234 INFO (SUITE-TestTlogReplica-seed#[E99EB240D3F155EC]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:42069 42069
[junit4] 2> 5666343 INFO (Thread-5913) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:42069 42069
[junit4] 2> 5666344 WARN (Thread-5913) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 32 /solr/configs/conf
[junit4] 2> 10 /solr/aliases.json
[junit4] 2> 9 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 10 /solr/clusterprops.json
[junit4] 2> 10 /solr/clusterstate.json
[junit4] 2> 6 /solr/collections/tlog_replica_test_recovery/state.json
[junit4] 2> 5 /solr/overseer_elect/election/98883842714501134-127.0.0.1:46588_solr-n_0000000003
[junit4] 2> 5 /solr/collections/tlog_replica_test_remove_leader/state.json
[junit4] 2> 4 /solr/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json
[junit4] 2> 4 /solr/collections/tlog_replica_test_kill_leader/state.json
[junit4] 2> 4 /solr/autoscaling.json
[junit4] 2> 4 /solr/collections/tlog_replica_test_recovery/leader_elect/shard1/election/98883842714501134-core_node4-n_0000000000
[junit4] 2> 4 /solr/collections/tlog_replica_test_basic_leader_election/state.json
[junit4] 2> 4 /solr/collections/tlog_replica_test_create_delete/state.json
[junit4] 2> 4 /solr/collections/tlog_replica_test_kill_tlog_replica/state.json
[junit4] 2> 3 /solr/collections/tlog_replica_test_create_delete/leader_elect/shard2/election/98883842714501134-core_node11-n_0000000000
[junit4] 2> 3 /solr/collections/tlog_replica_test_add_remove_tlog_replica/state.json
[junit4] 2> 3 /solr/collections/tlog_replica_test_only_leader_indexes/state.json
[junit4] 2> 3 /solr/collections/tlog_replica_test_delete_by_id/state.json
[junit4] 2> 2 /solr/collections/tlog_replica_test_create_delete/leader_elect/shard1/election/98883842714501134-core_node7-n_0000000001
[junit4] 2> 2 /solr/collections/tlog_replica_test_add_docs/state.json
[junit4] 2> 2 /solr/collections/tlog_replica_test_create_delete/leader_elect/shard2/election/98883842714501149-core_node10-n_0000000001
[junit4] 2> 2 /solr/collections/tlog_replica_test_real_time_get/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 10 /solr/live_nodes
[junit4] 2> 10 /solr/collections
[junit4] 2> 4 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 4 /solr/overseer/collection-queue-work
[junit4] 2> 4 /solr/overseer/queue-work
[junit4] 2> 3 /solr/overseer/queue
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_E99EB240D3F155EC-001
[junit4] 2> Oct 24, 2017 12:24:33 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {foo=Lucene50(blocksize=128), title_s=PostingsFormat(name=Memory), id=PostingsFormat(name=LuceneFixedGap), foo_s=Lucene50(blocksize=128)}, docValues:{_version_=DocValuesFormat(name=Lucene70), id=DocValuesFormat(name=Memory), inplace_updatable_int=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=994, maxMBSortInHeap=7.221330154553551, sim=RandomSimilarity(queryNorm=false): {}, locale=el-CY, timezone=Africa/Bissau
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=178639080,total=518520832
[junit4] 2> NOTE: All tests run in this JVM: [TimeZoneUtilsTest, TestFieldCacheSortRandom, SuggestComponentContextFilterQueryTest, XsltUpdateRequestHandlerTest, TestGraphMLResponseWriter, HdfsBasicDistributedZkTest, BadComponentTest, CustomCollectionTest, DefaultValueUpdateProcessorTest, CoreSorterTest, TestFuzzyAnalyzedSuggestions, ZkStateReaderTest, TestRequestStatusCollectionAPI, SortByFunctionTest, TestFunctionQuery, SubstringBytesRefFilterTest, TestCoreContainer, TestClassicSimilarityFactory, TestNumericTokenStream, TestGraphTermsQParserPlugin, TestPolicyCloud, TestShortCircuitedRequests, RemoteQueryErrorTest, TestDynamicLoading, TestMultiWordSynonyms, TestImpersonationWithHadoopAuth, TestUnifiedSolrHighlighter, DistanceUnitsTest, LoggingHandlerTest, TestPullReplicaErrorHandling, RequestLoggingTest, TestPerFieldSimilarityWithDefaultOverride, TestExpandComponent, HdfsAutoAddReplicasIntegrationTest, TestLRUCache, TestConfigReload, TestMinMaxOnMultiValuedField, TestLMJelinekMercerSimilarityFactory, DistributedFacetPivotWhiteBoxTest, PKIAuthenticationIntegrationTest, TestLegacyFieldReuse, SpellCheckCollatorWithCollapseTest, TestLegacyFieldCache, HttpSolrCallGetCoreTest, SpellPossibilityIteratorTest, RequiredFieldsTest, TestCopyFieldCollectionResource, TestCloudPseudoReturnFields, TestDownShardTolerantSearch, AutoscalingHistoryHandlerTest, ConfigSetsAPITest, TestGroupingSearch, TestCoreDiscovery, TestRuleBasedAuthorizationPlugin, TestXmlQParserPlugin, SolrCoreCheckLockOnStartupTest, MissingSegmentRecoveryTest, TestSolrDeletionPolicy2, SolrMetricReporterTest, TestSolrXml, ClusterStateTest, DocValuesMissingTest, RecoveryZkTest, WrapperMergePolicyFactoryTest, SolrGraphiteReporterTest, MoveReplicaTest, TestConfigSets, CdcrUpdateLogTest, ChaosMonkeyNothingIsSafeTest, SaslZkACLProviderTest, TestRecoveryHdfs, RankQueryTest, TestSearchPerf, AutoScalingHandlerTest, DistributedExpandComponentTest, TestFiltering, TestRawResponseWriter, SuggesterWFSTTest, OverseerTest, TestMultiValuedNumericRangeQuery, TestFieldCache, FieldMutatingUpdateProcessorTest, UnloadDistributedZkTest, ScriptEngineTest, TestCollapseQParserPlugin, SparseHLLTest, TestSchemaManager, DistribJoinFromCollectionTest, TestSolrCoreParser, CollectionsAPIAsyncDistributedZkTest, TestDistribIDF, QueryElevationComponentTest, TestLRUStatsCache, SolrIndexMetricsTest, TestElisionMultitermQuery, JSONWriterTest, TestZkChroot, CoreAdminHandlerTest, CollectionReloadTest, TestLMDirichletSimilarityFactory, TestCollationFieldDocValues, TestSolrQueryResponse, TestQueryTypes, TestNamedUpdateProcessors, DocValuesNotIndexedTest, TestStreamBody, DistributedSpellCheckComponentTest, ConnectionReuseTest, TestStressCloudBlindAtomicUpdates, HdfsCollectionsAPIDistributedZkTest, SystemInfoHandlerTest, TestRebalanceLeaders, FullSolrCloudDistribCmdsTest, SuggesterTest, AtomicUpdateProcessorFactoryTest, TestBulkSchemaConcurrent, TestLeaderElectionWithEmptyReplica, ZkNodePropsTest, TestTolerantSearch, TestFieldCollectionResource, DistributedFacetPivotLongTailTest, DistanceFunctionTest, BasicDistributedZkTest, DirectUpdateHandlerTest, TestIndexSearcher, SimplePostToolTest, AnalysisAfterCoreReloadTest, TestExtendedDismaxParser, DocValuesTest, TestSurroundQueryParser, TestQueryUtils, FileBasedSpellCheckerTest, PathHierarchyTokenizerFactoryTest, MoreLikeThisHandlerTest, IndexSchemaTest, TestCSVResponseWriter, QueryParsingTest, UniqFieldsUpdateProcessorFactoryTest, CSVRequestHandlerTest, TestQuerySenderListener, AlternateDirectoryTest, CopyFieldTest, TestMergePolicyConfig, MultiTermTest, TestDocSet, TestConfig, ExternalFileFieldSortTest, TestPostingsSolrHighlighter, TestCharFilters, EchoParamsTest, TestIBSimilarityFactory, TestFastOutputStream, OpenExchangeRatesOrgProviderTest, PluginInfoTest, PrimUtilsTest, SpellingQueryConverterTest, TestSolrJ, TestDocumentBuilder, FileUtilsTest, DistributedMLTComponentTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestDistributedMissingSort, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerConstructors, TestEmbeddedSolrServerSchemaAPI, TestJettySolrRunner, AliasIntegrationTest, AsyncCallRequestStatusResponseTest, CdcrReplicationDistributedZkTest, CdcrReplicationHandlerTest, CdcrVersionReplicationTest, CleanupOldIndexTest, ConcurrentDeleteAndCreateCollectionTest, CreateCollectionCleanupTest, DeleteLastCustomShardedReplicaTest, DeleteNodeTest, DeleteStatusTest, DistribCursorPagingTest, LeaderInitiatedRecoveryOnShardRestartTest, MoveReplicaHDFSFailoverTest, OverseerStatusTest, PeerSyncReplicationTest, SolrCLIZkUtilsTest, SolrXmlInZkTest, TestClusterProperties, TestCollectionAPI, TestCollectionsAPIViaSolrCloudCluster, TestConfigSetsAPIZkFailure, TestCryptoKeys, TestExclusionRuleCollectionAccess, TestLeaderInitiatedRecoveryThread, TestOnReconnectListenerSupport, TestPullReplica, TestRandomFlRTGCloud, TestRequestForwarding, TestSegmentSorting, TestSolrCloudWithDelegationTokens, TestTlogReplica]
[junit4] Completed [565/744 (3!)] on J2 in 176.56s, 13 tests, 1 error <<< FAILURES!
[...truncated 47830 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 191 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/191/
6 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.BasicDistributedZkTest
Error Message:
118 threads leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest: 1) Thread[id=15078, name=qtp1220744071-15078, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=15125, name=watches-2817-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=15157, name=Scheduler-1212435602, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=15177, name=zkCallback-2823-thread-2, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=15186, name=zkCallback-2810-thread-6, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=15179, name=zkCallback-2823-thread-4, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=15213, name=coreZkRegister-4658-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=15131, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=15203, name=searcherExecutor-4697-thread-1, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=15076, name=qtp1220744071-15076, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=15158, name=watches-2824-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=15174, name=zkCallback-2816-thread-3, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=15175, name=zkCallback-2816-thread-4, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=15075, name=qtp1220744071-15075, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=15105, name=qtp367347145-15105, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=15169, name=zkCallback-2804-thread-5, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=15215, name=recoveryExecutor-2802-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=15042, name=qtp990776779-15042, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Object.wait(Native Method) at org.apache.solr.cloud.OverseerTaskQueue$LatchWatcher.await(OverseerTaskQueue.java:149) at org.apache.solr.cloud.OverseerTaskQueue.offer(OverseerTaskQueue.java:199) at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:300) at org.apache.solr.handler.admin.CollectionsHandler.invokeAction(CollectionsHandler.java:244) at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:222) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) at org.eclipse.jetty.server.Server.handle(Server.java:534) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=15194, name=zkCallback-2810-thread-1-processing-n:127.0.0.1:53790_ufox%2Fj-SendThread(127.0.0.1:39256), state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 20) Thread[id=15172, name=zkCallback-2810-thread-4, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=15192, name=zkCallback-2816-thread-3-processing-n:127.0.0.1:40195_ufox%2Fj-SendThread(127.0.0.1:39256), state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 22) Thread[id=15114, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=15108, name=qtp367347145-15108, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=15049, name=qtp990776779-15049, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=15195, name=zkCallback-2810-thread-1-processing-n:127.0.0.1:53790_ufox%2Fj-EventThread, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 26) Thread[id=15113, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=15137, name=qtp183297797-15137-acceptor-0@51de79c-ServerConnector@63305544{HTTP/1.1,[http/1.1]}{127.0.0.1:53551}, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 28) Thread[id=15050, name=org.eclipse.jetty.server.session.HashSessionManager@ad95060Timer, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=15101, name=qtp367347145-15101, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=15102, name=qtp367347145-15102, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 31) Thread[id=15103, name=qtp367347145-15103-acceptor-0@349ab6a8-ServerConnector@359b5045{HTTP/1.1,[http/1.1]}{127.0.0.1:40195}, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=15234, name=zkCallback-2823-thread-7, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=15204, name=coreZkRegister-4669-thread-2, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 34) Thread[id=15182, name=zkCallback-2816-thread-6, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=15046, name=qtp990776779-15046, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=15227, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 37) Thread[id=15045, name=qtp990776779-15045, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 38) Thread[id=15154, name=zkCallback-2823-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 39) Thread[id=15233, name=zkCallback-2804-thread-8, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 40) Thread[id=15055, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 41) Thread[id=15229, name=ScheduledTrigger-4705-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:389) at org.apache.solr.cloud.ZkDistributedQueue.offer(ZkDistributedQueue.java:293) at org.apache.solr.cloud.autoscaling.TriggerEventQueue.offerEvent(TriggerEventQueue.java:58) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.enqueue(ScheduledTriggers.java:411) at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$3(ScheduledTriggers.java:205) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$145/1884738894.process(Unknown Source) at org.apache.solr.cloud.autoscaling.NodeLostTrigger.run(NodeLostTrigger.java:163) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.run(ScheduledTriggers.java:453) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 42) Thread[id=15173, name=zkCallback-2816-thread-2, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 43) Thread[id=15109, name=org.eclipse.jetty.server.session.HashSessionManager@392a8cc4Timer, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 44) Thread[id=15120, name=zkCallback-2816-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 45) Thread[id=15142, name=qtp183297797-15142, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 46) Thread[id=15207, name=updateExecutor-2820-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 47) Thread[id=15107, name=qtp367347145-15107, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 48) Thread[id=15232, name=zkCallback-2816-thread-7, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 49) Thread[id=15148, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 50) Thread[id=15121, name=Thread-4550, state=WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 51) Thread[id=15143, name=org.eclipse.jetty.server.session.HashSessionManager@35deb332Timer, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 52) Thread[id=15092, name=Thread-4541, state=WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 53) Thread[id=15104, name=qtp367347145-15104, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 54) Thread[id=15071, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 55) Thread[id=15048, name=qtp990776779-15048, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 56) Thread[id=15141, name=qtp183297797-15141, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 57) Thread[id=15084, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 58) Thread[id=15054, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 59) Thread[id=15228, name=OverseerAutoScalingTriggerThread-98877173344305170-127.0.0.1:53551_ufox%2Fj-n_0000000004, state=WAITING, group=Overseer autoscaling triggers] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:161) at java.lang.Thread.run(Thread.java:748) 60) Thread[id=15138, name=qtp183297797-15138, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 61) Thread[id=15237, name=ScheduledTrigger-4705-thread-4, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 62) Thread[id=15085, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 63) Thread[id=15043, name=qtp990776779-15043, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 64) Thread[id=15196, name=zkCallback-2804-thread-3-processing-n:127.0.0.1:35580_ufox%2Fj-SendThread(127.0.0.1:39256), state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 65) Thread[id=15166, name=zkCallback-2804-thread-2, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 66) Thread[id=15098, name=searcherExecutor-4675-thread-1, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 67) Thread[id=15135, name=qtp183297797-15135, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 68) Thread[id=15062, name=Thread-4532, state=WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 69) Thread[id=15167, name=zkCallback-2804-thread-3, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 70) Thread[id=15170, name=zkCallback-2810-thread-2, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 71) Thread[id=15199, name=coreZkRegister-4680-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 72) Thread[id=15208, name=recoveryExecutor-2821-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 73) Thread[id=15231, name=ScheduledTrigger-4705-thread-2, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 74) Thread[id=15047, name=qtp990776779-15047-acceptor-0@5af199d1-ServerConnector@2bf37a8d{HTTP/1.1,[http/1.1]}{127.0.0.1:35580}, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 75) Thread[id=15091, name=zkCallback-2810-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 76) Thread[id=15077, name=qtp1220744071-15077, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 77) Thread[id=15100, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 78) Thread[id=15067, name=watches-2805-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 79) Thread[id=15183, name=zkCallback-2804-thread-6, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 80) Thread[id=15080, name=org.eclipse.jetty.server.session.HashSessionManager@2dc7e136Timer, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 81) Thread[id=15168, name=zkCallback-2804-thread-4, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 82) Thread[id=15197, name=zkCallback-2804-thread-3-processing-n:127.0.0.1:35580_ufox%2Fj-EventThread, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 83) Thread[id=15171, name=zkCallback-2810-thread-3, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 84) Thread[id=15096, name=watches-2811-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 85) Thread[id=15226, name=OverseerCollectionConfigSetProcessor-98877173344305170-127.0.0.1:53551_ufox%2Fj-n_0000000004, state=TIMED_WAITING, group=Overseer collection creation process.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 86) Thread[id=15136, name=qtp183297797-15136, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 87) Thread[id=15216, name=coreZkRegister-4658-thread-2, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 88) Thread[id=15139, name=qtp183297797-15139, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 89) Thread[id=15069, name=searcherExecutor-4664-thread-1, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 90) Thread[id=15061, name=zkCallback-2804-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 91) Thread[id=15184, name=zkCallback-2804-thread-7, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 92) Thread[id=15225, name=OverseerStateUpdate-98877173344305170-127.0.0.1:53551_ufox%2Fj-n_0000000004, state=TIMED_WAITING, group=Overseer state updater.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:435) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:202) at java.lang.Thread.run(Thread.java:748) 93) Thread[id=15180, name=zkCallback-2823-thread-5, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 94) Thread[id=15181, name=zkCallback-2823-thread-6, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 95) Thread[id=15193, name=zkCallback-2816-thread-3-processing-n:127.0.0.1:40195_ufox%2Fj-EventThread, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 96) Thread[id=15124, name=Scheduler-1024594496, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 97) Thread[id=15079, name=qtp1220744071-15079, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 98) Thread[id=15236, name=ScheduledTrigger-4705-thread-3, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 99) Thread[id=15106, name=qtp367347145-15106, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 100) Thread[id=15072, name=qtp1220744071-15072, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 101) Thread[id=15095, name=Scheduler-1597017116, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 102) Thread[id=15155, name=Thread-4559, state=WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 103) Thread[id=15185, name=zkCallback-2810-thread-5, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 104) Thread[id=15176, name=zkCallback-2816-thread-5, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 105) Thread[id=15073, name=qtp1220744071-15073, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 106) Thread[id=15152, name=TEST-BasicDistributedZkTest.test-seed#[E0ED00A999BBA103]-SendThread(127.0.0.1:39256), state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 107) Thread[id=15074, name=qtp1220744071-15074-acceptor-0@ea51341-ServerConnector@35fb7f9f{HTTP/1.1,[http/1.1]}{127.0.0.1:53790}, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 108) Thread[id=15178, name=zkCallback-2823-thread-3, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 109) Thread[id=15044, name=qtp990776779-15044, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 110) Thread[id=15127, name=searcherExecutor-4686-thread-1, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 111) Thread[id=15140, name=qtp183297797-15140, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 112) Thread[id=15064, name=Scheduler-238212817, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 113) Thread[id=15153, name=TEST-BasicDistributedZkTest.test-seed#[E0ED00A999BBA103]-EventThread, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[...truncated too long message...]
, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 66) Thread[id=15178, name=zkCallback-2823-thread-3, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 67) Thread[id=15127, name=searcherExecutor-4686-thread-1, state=WAITING, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 68) Thread[id=15140, name=qtp183297797-15140, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 69) Thread[id=15092, name=Thread-4541, state=WAITING, group=TGRP-BasicDistributedZkTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678)
[junit4] > 70) Thread[id=15064, name=Scheduler-238212817, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 71) Thread[id=15200, name=coreZkRegister-4680-thread-2, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 72) Thread[id=15214, name=updateExecutor-2801-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 73) Thread[id=15202, name=coreZkRegister-4669-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 74) Thread[id=15235, name=zkCallback-2823-thread-8, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([E0ED00A999BBA103]:0)
[junit4] Completed [352/744 (3!)] on J0 in 127.38s, 1 test, 3 errors <<< FAILURES!
[...truncated 48429 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 190 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/190/
4 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
Error Message:
8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest: 1) Thread[id=644, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=1038, name=zkCallback-113-thread-1-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 3) Thread[id=1037, name=zkCallback-113-thread-1-SendThread(127.0.0.1:57157), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest:
1) Thread[id=644, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=1038, name=zkCallback-113-thread-1-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
3) Thread[id=1037, name=zkCallback-113-thread-1-SendThread(127.0.0.1:57157), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
at __randomizedtesting.SeedInfo.seed([96FB05935602FDD4]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=1037, name=zkCallback-113-thread-1-SendThread(127.0.0.1:57157), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=1037, name=zkCallback-113-thread-1-SendThread(127.0.0.1:57157), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
at __randomizedtesting.SeedInfo.seed([96FB05935602FDD4]:0)
FAILED: org.apache.solr.client.solrj.io.stream.SelectWithEvaluatorsTest.testSelectWithEvaluatorsStream
Error Message:
Collection not found: collection1
Stack Trace:
org.apache.solr.common.SolrException: Collection not found: collection1
at __randomizedtesting.SeedInfo.seed([E076EBD28405821B:6C2E0AA9D97CD2B8]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:834)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
at org.apache.solr.client.solrj.io.stream.SelectWithEvaluatorsTest.cleanIndex(SelectWithEvaluatorsTest.java:83)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test
Error Message:
KeeperErrorCode = Session expired for /live_nodes
Stack Trace:
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /live_nodes
at __randomizedtesting.SeedInfo.seed([96FB05935602FDD4:1EAF3A49F8FE902C]:0)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1532)
at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:340)
at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:337)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:337)
at org.apache.solr.common.cloud.ZkStateReader.refreshLiveNodes(ZkStateReader.java:718)
at org.apache.solr.common.cloud.ZkStateReader.updateLiveNodes(ZkStateReader.java:386)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.logReplicaTypesReplicationInfo(AbstractFullDistribZkTestBase.java:2067)
at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:312)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11819 lines...]
[junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
[junit4] 2> 342156 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[96FB05935602FDD4]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/init-core-data-001
[junit4] 2> 342209 WARN (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[96FB05935602FDD4]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=25 numCloses=25
[junit4] 2> 342229 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[96FB05935602FDD4]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 342231 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[96FB05935602FDD4]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 342231 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[96FB05935602FDD4]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /eov/db
[junit4] 2> 342250 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.ChaosMonkeyNothingIsSafeWithPullReplicasTest Starting ChaosMonkey test with 2 shards and 6 nodes
[junit4] 2> 342251 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 342473 INFO (Thread-129) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 342473 INFO (Thread-129) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 342481 ERROR (Thread-129) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 342577 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.ZkTestServer start zk server on port:57157
[junit4] 2> 342794 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 342809 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 342810 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 342811 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 342812 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 342829 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 342832 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 342833 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 342854 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 342856 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 342873 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 342921 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 345578 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 345657 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@31cdc7c7{/eov/db,null,AVAILABLE}
[junit4] 2> 345659 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5ea941cf{HTTP/1.1,[http/1.1]}{127.0.0.1:58799}
[junit4] 2> 345659 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.Server Started @366528ms
[junit4] 2> 345659 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/tempDir-001/control/data, hostContext=/eov/db, hostPort=58799, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/control-001/cores}
[junit4] 2> 345659 ERROR (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 345659 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 345659 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 345659 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 345659 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-22T00:54:58.604Z
[junit4] 2> 346262 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 346262 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/control-001/solr.xml
[junit4] 2> 346444 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 346444 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 346461 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 346466 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57157/solr
[junit4] 2> 346919 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 346920 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:58799_eov%2Fdb
[junit4] 2> 346921 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.Overseer Overseer (id=98869817863176196-127.0.0.1:58799_eov%2Fdb-n_0000000000) starting
[junit4] 2> 348006 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58799_eov%2Fdb
[junit4] 2> 348069 INFO (OverseerStateUpdate-98869817863176196-127.0.0.1:58799_eov%2Fdb-n_0000000000) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 348893 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 348934 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 348935 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 348936 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/control-001/cores
[junit4] 2> 349520 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 349537 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57157/solr ready
[junit4] 2> 349558 INFO (qtp1833550070-605) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:58799_eov%252Fdb&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 349635 INFO (OverseerThreadFactory-230-thread-1-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 349974 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 349974 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 350117 INFO (zkCallback-105-thread-1-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 351129 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 351238 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 353516 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 353688 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 353689 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 353762 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 353762 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 353943 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=30.818359375, floorSegmentMB=1.5107421875, forceMergeDeletesPctAllowed=22.137981077966007, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8867546865563619
[junit4] 2> 354037 WARN (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 355165 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 355165 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 355166 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 355166 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 355168 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=87148644, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 355243 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4029d23f[control_collection_shard1_replica_n1] main]
[junit4] 2> 355293 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 355294 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 355295 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 355449 INFO (searcherExecutor-233-thread-1-processing-n:127.0.0.1:58799_eov%2Fdb x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@4029d23f[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 355449 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581917099412946944
[junit4] 2> 355505 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 355505 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 355505 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:58799/eov/db/control_collection_shard1_replica_n1/
[junit4] 2> 355505 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 355505 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:58799/eov/db/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 355505 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 355520 INFO (zkCallback-105-thread-2-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 355521 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:58799/eov/db/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 355521 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 355578 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=5604
[junit4] 2> 355637 INFO (qtp1833550070-605) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 355681 INFO (zkCallback-105-thread-2-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 356647 INFO (qtp1833550070-605) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:58799_eov%252Fdb&wt=javabin&version=2} status=0 QTime=7089
[junit4] 2> 357015 INFO (OverseerCollectionConfigSetProcessor-98869817863176196-127.0.0.1:58799_eov%2Fdb-n_0000000000) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 357020 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 357021 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57157/solr ready
[junit4] 2> 357021 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 357026 INFO (qtp1833550070-606) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 357094 INFO (OverseerThreadFactory-230-thread-2-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 357113 WARN (OverseerThreadFactory-230-thread-2-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 357253 INFO (qtp1833550070-606) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 357253 INFO (qtp1833550070-606) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=227
[junit4] 2> 357276 INFO (OverseerCollectionConfigSetProcessor-98869817863176196-127.0.0.1:58799_eov%2Fdb-n_0000000000) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 359123 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-1-001 of type NRT
[junit4] 2> 359138 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 359223 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@ed91ae4{/eov/db,null,AVAILABLE}
[junit4] 2> 359223 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2c089a0f{HTTP/1.1,[http/1.1]}{127.0.0.1:47004}
[junit4] 2> 359223 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.Server Started @380092ms
[junit4] 2> 359223 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/eov/db, hostPort=47004, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-1-001/cores}
[junit4] 2> 359223 ERROR (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 359224 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 359224 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 359224 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 359224 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-22T00:55:12.169Z
[junit4] 2> 359269 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 359269 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-1-001/solr.xml
[junit4] 2> 359272 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 359272 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 359494 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 359505 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57157/solr
[junit4] 2> 359718 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 359721 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 359733 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47004_eov%2Fdb
[junit4] 2> 359734 INFO (zkCallback-105-thread-2-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 359805 INFO (zkCallback-113-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 361115 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 361241 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 361241 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 361243 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-1-001/cores
[junit4] 2> 361626 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 361986 INFO (qtp1297987795-652) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:47004_eov%252Fdb&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 362016 INFO (OverseerThreadFactory-230-thread-3-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:47004_eov%2Fdb for creating new replica
[junit4] 2> 362183 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 362184 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 362275 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 363375 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 363618 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema [collection1_shard2_replica_n41] Schema name=test
[junit4] 2> 364606 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 366350 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n41' using configuration from collection collection1, trusted=true
[junit4] 2> 366351 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n41' (registry 'solr.core.collection1.shard2.replica_n41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 366351 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 366351 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [[collection1_shard2_replica_n41] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-1-001/cores/collection1_shard2_replica_n41], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-1-001/cores/collection1_shard2_replica_n41/data/]
[junit4] 2> 366394 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=30.818359375, floorSegmentMB=1.5107421875, forceMergeDeletesPctAllowed=22.137981077966007, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8867546865563619
[junit4] 2> 367348 WARN (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 367815 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 367815 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 367816 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 367816 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 367849 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=87148644, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 368050 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.SolrIndexSearcher Opening [Searcher@1ecc0a46[collection1_shard2_replica_n41] main]
[junit4] 2> 368052 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 368052 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 368053 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 368055 INFO (searcherExecutor-244-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb x:collection1_shard2_replica_n41 s:shard2 c:collection1) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [collection1_shard2_replica_n41] Registered new searcher Searcher@1ecc0a46[collection1_shard2_replica_n41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 368055 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581917112631296000
[junit4] 2> 368060 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 368060 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 368060 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:47004/eov/db/collection1_shard2_replica_n41/
[junit4] 2> 368060 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 368060 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy http://127.0.0.1:47004/eov/db/collection1_shard2_replica_n41/ has no replicas
[junit4] 2> 368060 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 368064 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:47004/eov/db/collection1_shard2_replica_n41/ shard2
[junit4] 2> 368113 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 368114 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 368115 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=5932
[junit4] 2> 368119 INFO (qtp1297987795-652) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:47004_eov%252Fdb&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=6132
[junit4] 2> 368217 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 368353 INFO (OverseerCollectionConfigSetProcessor-98869817863176196-127.0.0.1:58799_eov%2Fdb-n_0000000000) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 370037 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-2-001 of type NRT
[junit4] 2> 370042 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 370046 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4b799471{/eov/db,null,AVAILABLE}
[junit4] 2> 370046 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@18c6e2e4{HTTP/1.1,[http/1.1]}{127.0.0.1:50167}
[junit4] 2> 370046 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.Server Started @390915ms
[junit4] 2> 370046 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/eov/db, hostPort=50167, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-2-001/cores}
[junit4] 2> 370046 ERROR (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 370085 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 370085 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 370085 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 370085 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-22T00:55:23.030Z
[junit4] 2> 370142 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 370142 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-2-001/solr.xml
[junit4] 2> 370146 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 370146 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 370147 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 370174 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57157/solr
[junit4] 2> 370285 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 370290 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 370308 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50167_eov%2Fdb
[junit4] 2> 370308 INFO (zkCallback-105-thread-2-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 370309 INFO (zkCallback-113-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 370325 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 370445 INFO (zkCallback-124-thread-2-processing-n:127.0.0.1:50167_eov%2Fdb) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 371131 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 371230 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 371230 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 371232 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-2-001/cores
[junit4] 2> 372110 INFO (qtp1297987795-653) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:50167_eov%252Fdb&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 372144 INFO (OverseerThreadFactory-230-thread-4-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:50167_eov%2Fdb for creating new replica
[junit4] 2> 372270 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 372270 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 372289 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 372375 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 372575 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.IndexSchema [collection1_shard1_replica_n43] Schema name=test
[junit4] 2> 373764 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 374166 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n43' using configuration from collection collection1, trusted=true
[junit4] 2> 374227 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n43' (registry 'solr.core.collection1.shard1.replica_n43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 374227 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 374228 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrCore [[collection1_shard1_replica_n43] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-2-001/cores/collection1_shard1_replica_n43], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-2-001/cores/collection1_shard1_replica_n43/data/]
[junit4] 2> 374255 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=30.818359375, floorSegmentMB=1.5107421875, forceMergeDeletesPctAllowed=22.137981077966007, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8867546865563619
[junit4] 2> 374358 WARN (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 376057 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 376057 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 376058 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 376058 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 376060 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=87148644, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 376145 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.SolrIndexSearcher Opening [Searcher@5182bff6[collection1_shard1_replica_n43] main]
[junit4] 2> 376157 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 376158 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 376158 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 376170 INFO (searcherExecutor-255-thread-1-processing-n:127.0.0.1:50167_eov%2Fdb x:collection1_shard1_replica_n43 s:shard1 c:collection1) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrCore [collection1_shard1_replica_n43] Registered new searcher Searcher@5182bff6[collection1_shard1_replica_n43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 376173 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581917121143635968
[junit4] 2> 376195 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 376195 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 376195 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:50167/eov/db/collection1_shard1_replica_n43/
[junit4] 2> 376195 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 376195 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy http://127.0.0.1:50167/eov/db/collection1_shard1_replica_n43/ has no replicas
[junit4] 2> 376195 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 376217 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 376234 INFO (zkCallback-124-thread-2-processing-n:127.0.0.1:50167_eov%2Fdb) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 376235 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:50167/eov/db/collection1_shard1_replica_n43/ shard1
[junit4] 2> 376235 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 376253 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3983
[junit4] 2> 376270 INFO (qtp1297987795-653) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:50167_eov%252Fdb&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=4160
[junit4] 2> 376381 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 376382 INFO (zkCallback-124-thread-2-processing-n:127.0.0.1:50167_eov%2Fdb) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 377075 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-3-001 of type NRT
[junit4] 2> 377112 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 377157 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@290ab17c{/eov/db,null,AVAILABLE}
[junit4] 2> 377158 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@ceeb05c{HTTP/1.1,[http/1.1]}{127.0.0.1:42402}
[junit4] 2> 377158 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.Server Started @398027ms
[junit4] 2> 377158 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/eov/db, hostPort=42402, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-3-001/cores}
[junit4] 2> 377158 ERROR (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 377250 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 377250 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 377250 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 377250 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-22T00:55:30.195Z
[junit4] 2> 377327 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 377327 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-3-001/solr.xml
[junit4] 2> 377331 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 377331 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 377332 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 377434 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57157/solr
[junit4] 2> 377565 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 377567 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 377568 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42402_eov%2Fdb
[junit4] 2> 377577 INFO (zkCallback-124-thread-2-processing-n:127.0.0.1:50167_eov%2Fdb) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 377593 INFO (zkCallback-113-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 377578 INFO (zkCallback-105-thread-1-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 377577 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 377757 INFO (zkCallback-130-thread-2-processing-n:127.0.0.1:42402_eov%2Fdb) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 378141 INFO (OverseerCollectionConfigSetProcessor-98869817863176196-127.0.0.1:58799_eov%2Fdb-n_0000000000) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 379334 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 380496 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 380496 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 380554 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-3-001/cores
[junit4] 2> 381138 INFO (qtp1841800575-713) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:42402_eov%252Fdb&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 381199 INFO (OverseerThreadFactory-230-thread-5-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:42402_eov%2Fdb for creating new replica
[junit4] 2> 381246 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 381253 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 381286 INFO (zkCallback-124-thread-2-processing-n:127.0.0.1:50167_eov%2Fdb) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 381286 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 381301 INFO (zkCallback-130-thread-2-processing-n:127.0.0.1:42402_eov%2Fdb) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 382769 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 382911 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.s.IndexSchema [collection1_shard2_replica_n45] Schema name=test
[junit4] 2> 384076 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 384181 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n45' using configuration from collection collection1, trusted=true
[junit4] 2> 384257 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n45' (registry 'solr.core.collection1.shard2.replica_n45') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 384257 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 384257 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.SolrCore [[collection1_shard2_replica_n45] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-3-001/cores/collection1_shard2_replica_n45], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-3-001/cores/collection1_shard2_replica_n45/data/]
[junit4] 2> 384282 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=30.818359375, floorSegmentMB=1.5107421875, forceMergeDeletesPctAllowed=22.137981077966007, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8867546865563619
[junit4] 2> 384373 WARN (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 385117 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 385117 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 385118 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 385118 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 385120 INFO (qtp1841800575-711
[...truncated too long message...]
kStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 622607 INFO (zkCallback-139-thread-2-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 622705 INFO (coreCloseExecutor-509-thread-1) [n:127.0.0.1:36943_solr c:collection1_collection s:shard2 r:core_node4 x:collection1_collection_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1_collection.shard2.leader, tag=576650243
[junit4] 2> 622742 INFO (zkCallback-139-thread-3-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 622742 INFO (zkCallback-145-thread-1-processing-n:127.0.0.1:49113_solr) [n:127.0.0.1:49113_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 622742 INFO (zkCallback-145-thread-3-processing-n:127.0.0.1:49113_solr) [n:127.0.0.1:49113_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 622747 INFO (zkCallback-143-thread-2-processing-n:127.0.0.1:54044_solr) [n:127.0.0.1:54044_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (0)
[junit4] 2> 622750 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f41cb1f1e000a, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 622756 INFO (zkCallback-139-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 622790 INFO (jetty-closer-124-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@10ae52e9{/solr,null,UNAVAILABLE}
[junit4] 2> 622856 INFO (coreCloseExecutor-511-thread-1) [n:127.0.0.1:49113_solr c:collection1_collection s:shard1 r:core_node3 x:collection1_collection_shard1_replica_n1] o.a.s.c.SolrCore [collection1_collection_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@27c1338e
[junit4] 2> 622856 INFO (coreCloseExecutor-511-thread-1) [n:127.0.0.1:49113_solr c:collection1_collection s:shard1 r:core_node3 x:collection1_collection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1_collection.shard1.replica_n1, tag=666973070
[junit4] 2> 622856 INFO (coreCloseExecutor-511-thread-1) [n:127.0.0.1:49113_solr c:collection1_collection s:shard1 r:core_node3 x:collection1_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3e64835: rootName = solr_49113, domain = solr.core.collection1_collection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.collection1_collection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@6900f402
[junit4] 2> 622918 INFO (zkCallback-145-thread-2-processing-n:127.0.0.1:49113_solr) [n:127.0.0.1:49113_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 622919 INFO (zkCallback-139-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 622986 INFO (zkCallback-145-thread-3-processing-n:127.0.0.1:49113_solr) [n:127.0.0.1:49113_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 623041 INFO (zkCallback-139-thread-3-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 623042 WARN (zkCallback-139-thread-3-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /collections/collection1_collection/state.json]
[junit4] 2> 623056 INFO (coreCloseExecutor-511-thread-1) [n:127.0.0.1:49113_solr c:collection1_collection s:shard1 r:core_node3 x:collection1_collection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1_collection.shard1.leader, tag=666973070
[junit4] 2> 623070 INFO (jetty-closer-124-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@59eeb86c{/solr,null,UNAVAILABLE}
[junit4] 2> 623201 INFO (jetty-closer-124-thread-2) [ ] o.a.s.c.Overseer Overseer (id=98870057481666572-127.0.0.1:49113_solr-n_0000000000) closing
[junit4] 2> 623201 INFO (OverseerStateUpdate-98870057481666572-127.0.0.1:49113_solr-n_0000000000) [n:127.0.0.1:49113_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:49113_solr
[junit4] 2> 623299 WARN (OverseerAutoScalingTriggerThread-98870057481666572-127.0.0.1:49113_solr-n_0000000000) [n:127.0.0.1:49113_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 623356 INFO (jetty-closer-124-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@183525b4{/solr,null,UNAVAILABLE}
[junit4] 2> 624294 ERROR (SUITE-SelectWithEvaluatorsTest-seed#[E076EBD28405821B]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 624294 INFO (SUITE-SelectWithEvaluatorsTest-seed#[E076EBD28405821B]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39692 39692
[junit4] 2> 624911 INFO (Thread-243) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39692 39692
[junit4] 2> 624911 WARN (Thread-243) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/aliases.json
[junit4] 2> 5 /solr/clusterprops.json
[junit4] 2> 4 /solr/security.json
[junit4] 2> 2 /solr/configs/conf
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 16 /solr/collections/collection1_collection/state.json
[junit4] 2> 5 /solr/clusterstate.json
[junit4] 2> 2 /solr/overseer_elect/election/98870057481666572-127.0.0.1:49113_solr-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/live_nodes
[junit4] 2> 5 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.SelectWithEvaluatorsTest_E076EBD28405821B-001
[junit4] 2> Oct 22, 2017 1:56:17 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1672, maxMBSortInHeap=6.692899709764929, sim=RandomSimilarity(queryNorm=true): {}, locale=mk-MK, timezone=Etc/GMT-5
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=283037352,total=470286336
[junit4] 2> NOTE: All tests run in this JVM: [SolrQueryTest, TestCollectionAdminRequest, NormalizeEvaluatorTest, TestJsonRecordReader, HyperbolicSineEvaluatorTest, PowerEvaluatorTest, JdbcDriverTest, TestSolrJErrorHandling, CloudSolrClientTest, QueryResponseTest, UuidEvaluatorTest, RegressionEvaluatorTest, TestDocumentObjectBinder, GetByIdTest, CommonParamsTest, LessThanEqualToEvaluatorTest, TestSuggesterResponse, FacetFieldTest, ClientUtilsTest, SolrExampleEmbeddedTest, TestValidatingJsonMap, LBHttpSolrClientBuilderTest, TestCloudSolrClientConnections, LargeVolumeBinaryJettyTest, LargeVolumeEmbeddedTest, TestSolrProperties, MergeIndexesEmbeddedTest, NamedListTest, AnlysisResponseBaseTest, TestLBHttpSolrClient, SelectWithEvaluatorsTest]
[junit4] Completed [101/145 (1!)] on J1 in 26.95s, 1 test, 1 error <<< FAILURES!
[...truncated 46436 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 189 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/189/
6 tests failed.
FAILED: org.apache.solr.cloud.CdcrBootstrapTest.testBootstrapWithSourceCluster
Error Message:
Captured an uncaught exception in thread: Thread[id=848, name=Thread-154, state=RUNNABLE, group=TGRP-CdcrBootstrapTest]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=848, name=Thread-154, state=RUNNABLE, group=TGRP-CdcrBootstrapTest]
at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4:E4158E6F316E538E]:0)
Caused by: java.lang.AssertionError: 1
at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4]:0)
at org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:192)
at org.apache.solr.core.SolrCore.close(SolrCore.java:1614)
at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:870)
at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1285)
at org.apache.solr.handler.IndexFetcher.lambda$reloadCore$0(IndexFetcher.java:910)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.CdcrBootstrapTest
Error Message:
ObjectTracker found 3 object(s) that were not released!!! [SolrCore, MockDirectoryWrapper, RawDirectoryWrapper] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1020) at org.apache.solr.core.SolrCore.reload(SolrCore.java:637) at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1284) at org.apache.solr.handler.IndexFetcher.lambda$reloadCore$0(IndexFetcher.java:910) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:494) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:338) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:421) at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:768) at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:723) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:92) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:742) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:935) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:844) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1036) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:948) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) at org.eclipse.jetty.server.Server.handle(Server.java:534) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 3 object(s) that were not released!!! [SolrCore, MockDirectoryWrapper, RawDirectoryWrapper]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1020)
at org.apache.solr.core.SolrCore.reload(SolrCore.java:637)
at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1284)
at org.apache.solr.handler.IndexFetcher.lambda$reloadCore$0(IndexFetcher.java:910)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:494)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:338)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:421)
at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:768)
at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:723)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:92)
at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:742)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:935)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:844)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1036)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:948)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:534)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:295)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestHdfsCloudBackupRestore.test
Error Message:
expected:<COMPLETED> but was:<RUNNING>
Stack Trace:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>
at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4:B517E0719CF62D3C]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:277)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.testSslAndClientAuth
Error Message:
Could not find collection:first_collection
Stack Trace:
java.lang.AssertionError: Could not find collection:first_collection
at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4:EEC73666A0C71B38]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNotNull(Assert.java:526)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.checkCreateCollection(TestMiniSolrCloudClusterSSL.java:202)
at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.checkClusterWithCollectionCreations(TestMiniSolrCloudClusterSSL.java:172)
at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.checkClusterWithNodeReplacement(TestMiniSolrCloudClusterSSL.java:138)
at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.testSslAndClientAuth(TestMiniSolrCloudClusterSSL.java:120)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestMiniSolrCloudClusterSSL
Error Message:
9 threads leaked from SUITE scope at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL: 1) Thread[id=9145, name=OverseerAutoScalingTriggerThread-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=WAITING, group=Overseer autoscaling triggers] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:161) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=9053, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=9146, name=ScheduledTrigger-4119-thread-1, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:389) at org.apache.solr.cloud.ZkDistributedQueue.offer(ZkDistributedQueue.java:293) at org.apache.solr.cloud.autoscaling.TriggerEventQueue.offerEvent(TriggerEventQueue.java:58) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.enqueue(ScheduledTriggers.java:411) at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$3(ScheduledTriggers.java:205) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$237/524708820.process(Unknown Source) at org.apache.solr.cloud.autoscaling.NodeLostTrigger.run(NodeLostTrigger.java:163) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.run(ScheduledTriggers.java:453) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=9076, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=9143, name=OverseerStateUpdate-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=TIMED_WAITING, group=Overseer state updater.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:435) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:202) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=9144, name=OverseerCollectionConfigSetProcessor-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=TIMED_WAITING, group=Overseer collection creation process.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=9108, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-EventThread, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 8) Thread[id=9054, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=9107, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-SendThread(127.0.0.1:48343), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL:
1) Thread[id=9145, name=OverseerAutoScalingTriggerThread-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=WAITING, group=Overseer autoscaling triggers]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:161)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=9053, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=9146, name=ScheduledTrigger-4119-thread-1, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:389)
at org.apache.solr.cloud.ZkDistributedQueue.offer(ZkDistributedQueue.java:293)
at org.apache.solr.cloud.autoscaling.TriggerEventQueue.offerEvent(TriggerEventQueue.java:58)
at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.enqueue(ScheduledTriggers.java:411)
at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$3(ScheduledTriggers.java:205)
at org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$237/524708820.process(Unknown Source)
at org.apache.solr.cloud.autoscaling.NodeLostTrigger.run(NodeLostTrigger.java:163)
at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.run(ScheduledTriggers.java:453)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=9076, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=9143, name=OverseerStateUpdate-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=TIMED_WAITING, group=Overseer state updater.]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:435)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:202)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=9144, name=OverseerCollectionConfigSetProcessor-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=TIMED_WAITING, group=Overseer collection creation process.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387)
at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=9108, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-EventThread, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
8) Thread[id=9054, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=9107, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-SendThread(127.0.0.1:48343), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestMiniSolrCloudClusterSSL
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=9146, name=ScheduledTrigger-4119-thread-1, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=9107, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-SendThread(127.0.0.1:48343), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=9146, name=ScheduledTrigger-4119-thread-1, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=9107, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-SendThread(127.0.0.1:48343), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4]:0)
Build Log:
[...truncated 11625 lines...]
[junit4] Suite: org.apache.solr.cloud.CdcrBootstrapTest
[junit4] 2> 268050 INFO (SUITE-CdcrBootstrapTest-seed#[3D43DFAB320A40C4]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_3D43DFAB320A40C4-001/init-core-data-001
[junit4] 2> 268050 WARN (SUITE-CdcrBootstrapTest-seed#[3D43DFAB320A40C4]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 268050 INFO (SUITE-CdcrBootstrapTest-seed#[3D43DFAB320A40C4]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 268072 INFO (SUITE-CdcrBootstrapTest-seed#[3D43DFAB320A40C4]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 268104 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testBootstrapWithContinousIndexingOnSourceCluster
[junit4] 2> 268105 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_3D43DFAB320A40C4-001/cdcr-target-001
[junit4] 2> 268106 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 268124 INFO (Thread-73) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 268124 INFO (Thread-73) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 268197 ERROR (Thread-73) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 268421 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.ZkTestServer start zk server on port:60334
[junit4] 2> 268532 INFO (jetty-launcher-60-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 268701 INFO (jetty-launcher-60-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1075f419{/solr,null,AVAILABLE}
[junit4] 2> 269676 INFO (jetty-launcher-60-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4046fad4{SSL,[ssl, http/1.1]}{127.0.0.1:37550}
[junit4] 2> 269676 INFO (jetty-launcher-60-thread-1) [ ] o.e.j.s.Server Started @288801ms
[junit4] 2> 269676 INFO (jetty-launcher-60-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37550}
[junit4] 2> 269676 ERROR (jetty-launcher-60-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 269676 INFO (jetty-launcher-60-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 269677 INFO (jetty-launcher-60-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 269677 INFO (jetty-launcher-60-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 269677 INFO (jetty-launcher-60-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-20T22:31:23.523Z
[junit4] 2> 269928 INFO (jetty-launcher-60-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 269978 INFO (jetty-launcher-60-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60334/solr
[junit4] 2> 270935 INFO (jetty-launcher-60-thread-1) [n:127.0.0.1:37550_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 270936 INFO (jetty-launcher-60-thread-1) [n:127.0.0.1:37550_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37550_solr
[junit4] 2> 270955 INFO (jetty-launcher-60-thread-1) [n:127.0.0.1:37550_solr ] o.a.s.c.Overseer Overseer (id=98863591063945219-127.0.0.1:37550_solr-n_0000000000) starting
[junit4] 2> 271128 INFO (jetty-launcher-60-thread-1) [n:127.0.0.1:37550_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37550_solr
[junit4] 2> 271154 INFO (OverseerStateUpdate-98863591063945219-127.0.0.1:37550_solr-n_0000000000) [n:127.0.0.1:37550_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 273473 INFO (jetty-launcher-60-thread-1) [n:127.0.0.1:37550_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37550.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e79c86e
[junit4] 2> 273535 INFO (jetty-launcher-60-thread-1) [n:127.0.0.1:37550_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37550.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e79c86e
[junit4] 2> 273536 INFO (jetty-launcher-60-thread-1) [n:127.0.0.1:37550_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37550.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e79c86e
[junit4] 2> 273576 INFO (jetty-launcher-60-thread-1) [n:127.0.0.1:37550_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_3D43DFAB320A40C4-001/cdcr-target-001/node1/.
[junit4] 1> Target zkHost = 127.0.0.1:60334/solr
[junit4] 2> 274166 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_3D43DFAB320A40C4-001/cdcr-source-001
[junit4] 2> 274166 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 274184 INFO (Thread-83) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 274184 INFO (Thread-83) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 274264 ERROR (Thread-83) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 274437 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.ZkTestServer start zk server on port:59846
[junit4] 2> 274522 INFO (jetty-launcher-73-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 274605 INFO (jetty-launcher-73-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@21592feb{/solr,null,AVAILABLE}
[junit4] 2> 274606 INFO (jetty-launcher-73-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@24a4020f{SSL,[ssl, http/1.1]}{127.0.0.1:60685}
[junit4] 2> 274606 INFO (jetty-launcher-73-thread-1) [ ] o.e.j.s.Server Started @293731ms
[junit4] 2> 274606 INFO (jetty-launcher-73-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=60685}
[junit4] 2> 274606 ERROR (jetty-launcher-73-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 274684 INFO (jetty-launcher-73-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 274684 INFO (jetty-launcher-73-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 274684 INFO (jetty-launcher-73-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 274684 INFO (jetty-launcher-73-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-20T22:31:28.530Z
[junit4] 2> 274724 INFO (jetty-launcher-73-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 275156 INFO (jetty-launcher-73-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59846/solr
[junit4] 2> 277452 INFO (jetty-launcher-73-thread-1) [n:127.0.0.1:60685_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 277452 INFO (jetty-launcher-73-thread-1) [n:127.0.0.1:60685_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60685_solr
[junit4] 2> 277453 INFO (jetty-launcher-73-thread-1) [n:127.0.0.1:60685_solr ] o.a.s.c.Overseer Overseer (id=98863591460044803-127.0.0.1:60685_solr-n_0000000000) starting
[junit4] 2> 277584 INFO (jetty-launcher-73-thread-1) [n:127.0.0.1:60685_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60685_solr
[junit4] 2> 277623 INFO (zkCallback-80-thread-1-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 278718 INFO (jetty-launcher-73-thread-1) [n:127.0.0.1:60685_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60685.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e79c86e
[junit4] 2> 278925 INFO (jetty-launcher-73-thread-1) [n:127.0.0.1:60685_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60685.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e79c86e
[junit4] 2> 278925 INFO (jetty-launcher-73-thread-1) [n:127.0.0.1:60685_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60685.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e79c86e
[junit4] 2> 278926 INFO (jetty-launcher-73-thread-1) [n:127.0.0.1:60685_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_3D43DFAB320A40C4-001/cdcr-source-001/node1/.
[junit4] 2> 279957 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 279958 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59846/solr ready
[junit4] 2> 280513 INFO (qtp183448557-392) [n:127.0.0.1:60685_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=cdcr-source&name=cdcr-source&nrtReplicas=1&action=CREATE&numShards=1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 280523 INFO (OverseerThreadFactory-145-thread-1-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.c.CreateCollectionCmd Create collection cdcr-source
[junit4] 2> 280846 INFO (OverseerStateUpdate-98863591460044803-127.0.0.1:60685_solr-n_0000000000) [n:127.0.0.1:60685_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"cdcr-source",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"cdcr-source_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:60685/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 281073 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=cdcr-source&newCollection=true&collection=cdcr-source&version=2&replicaType=NRT&coreNodeName=core_node2&name=cdcr-source_shard1_replica_n1&action=CREATE&numShards=1&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin
[junit4] 2> 281074 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 281249 INFO (zkCallback-80-thread-1-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 281269 INFO (zkCallback-80-thread-2-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 282374 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 282471 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
[junit4] 2> 282495 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.IndexSchema [cdcr-source_shard1_replica_n1] Schema name=minimal
[junit4] 2> 282569 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 282920 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ManagedIndexSchema Created and persisted managed schema znode at /configs/cdcr-source/managed-schema
[junit4] 2> 282924 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory After upgrading to managed schema in ZooKeeper, renamed the non-managed schema /configs/cdcr-source/schema.xml to /configs/cdcr-source/schema.xml.bak
[junit4] 2> 282924 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'cdcr-source_shard1_replica_n1' using configuration from collection cdcr-source, trusted=true
[junit4] 2> 282925 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60685.solr.core.cdcr-source.shard1.replica_n1' (registry 'solr.core.cdcr-source.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e79c86e
[junit4] 2> 282925 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 282925 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [[cdcr-source_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_3D43DFAB320A40C4-001/cdcr-source-001/node1/cdcr-source_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_3D43DFAB320A40C4-001/cdcr-source-001/node1/./cdcr-source_shard1_replica_n1/data/]
[junit4] 2> 284148 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 284148 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 284150 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 284150 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 284260 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@510baeec[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 284261 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/cdcr-source
[junit4] 2> 284262 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/cdcr-source
[junit4] 2> 284264 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/cdcr-source/managed-schema
[junit4] 2> 284282 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 284283 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 284400 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrBufferStateManager Created znode /collections/cdcr-source/cdcr/state/buffer
[junit4] 2> 284413 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrProcessStateManager Created znode /collections/cdcr-source/cdcr/state/process
[junit4] 2> 285240 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581817474932801536
[junit4] 2> 285297 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@510baeec[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 285301 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 285301 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 285301 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:60685/solr/cdcr-source_shard1_replica_n1/
[junit4] 2> 285301 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 285301 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:60685/solr/cdcr-source_shard1_replica_n1/ has no replicas
[junit4] 2> 285301 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 285333 INFO (zkCallback-80-thread-3-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ cdcr-source:shard1
[junit4] 2> 285353 INFO (zkCallback-80-thread-2-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 285353 INFO (zkCallback-80-thread-3-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 285355 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:60685/solr/cdcr-source_shard1_replica_n1/ shard1
[junit4] 2> 285355 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 285366 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=cdcr-source&newCollection=true&collection=cdcr-source&version=2&replicaType=NRT&coreNodeName=core_node2&name=cdcr-source_shard1_replica_n1&action=CREATE&numShards=1&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=4292
[junit4] 2> 285405 INFO (qtp183448557-392) [n:127.0.0.1:60685_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 285485 INFO (zkCallback-80-thread-2-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 285485 INFO (zkCallback-80-thread-3-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 286428 INFO (qtp183448557-392) [n:127.0.0.1:60685_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=cdcr-source&name=cdcr-source&nrtReplicas=1&action=CREATE&numShards=1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2} status=0 QTime=5915
[junit4] 1> Adding 10 docs with commit=true, numDocs=100
[junit4] 2> 286549 INFO (OverseerCollectionConfigSetProcessor-98863591460044803-127.0.0.1:60685_solr-n_0000000000) [n:127.0.0.1:60685_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 287011 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=494
[junit4] 2> 287064 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817476845404160,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 287065 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817476845404160
[junit4] 2> 287544 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4422f037[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 287546 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@4422f037[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100)))}
[junit4] 2> 287547 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 287547 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=482
[junit4] 1> Adding 10 docs with commit=true, numDocs=200
[junit4] 2> 287778 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=229
[junit4] 2> 287800 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817477617156096,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 287800 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817477617156096
[junit4] 2> 288412 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@44c22b12[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 288413 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 288414 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@44c22b12[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100)))}
[junit4] 2> 288415 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=615
[junit4] 1> Adding 10 docs with commit=true, numDocs=300
[junit4] 2> 288684 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=266
[junit4] 2> 288685 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817478545145856,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 288744 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817478545145856
[junit4] 2> 289112 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@404f72e6[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 289114 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@404f72e6[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100) Uninverting(_2(7.2.0):C100)))}
[junit4] 2> 289114 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 289115 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=429
[junit4] 1> Adding 10 docs with commit=true, numDocs=400
[junit4] 2> 289357 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=239
[junit4] 2> 289385 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817479279149056,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 289385 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817479279149056
[junit4] 2> 289684 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7056aaf3[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 289685 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 289686 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@7056aaf3[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100) Uninverting(_2(7.2.0):C100) Uninverting(_3(7.2.0):C100)))}
[junit4] 2> 289687 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=301
[junit4] 1> Adding 10 docs with commit=true, numDocs=500
[junit4] 2> 289790 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=101
[junit4] 2> 289805 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817479719550976,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 289805 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817479719550976
[junit4] 2> 290089 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5d1d5a6d[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 290089 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 290091 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@5d1d5a6d[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100) Uninverting(_2(7.2.0):C100) Uninverting(_3(7.2.0):C100) Uninverting(_4(7.2.0):C100)))}
[junit4] 2> 290091 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=286
[junit4] 1> Adding 10 docs with commit=true, numDocs=600
[junit4] 2> 290184 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=91
[junit4] 2> 290220 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817480154710016,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 290221 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817480154710016
[junit4] 2> 292633 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@49e4bdd5[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 292633 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 292635 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@49e4bdd5[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100) Uninverting(_2(7.2.0):C100) Uninverting(_3(7.2.0):C100) Uninverting(_4(7.2.0):C100) Uninverting(_5(7.2.0):C100)))}
[junit4] 2> 292635 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=2415
[junit4] 1> Adding 10 docs with commit=true, numDocs=700
[junit4] 2> 292674 INFO (qtp183448557-396) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=37
[junit4] 2> 292704 INFO (qtp183448557-396) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817482759372800,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 292704 INFO (qtp183448557-396) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817482759372800
[junit4] 2> 293037 INFO (qtp183448557-396) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3f062990[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 293037 INFO (qtp183448557-396) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 293039 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@3f062990[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100) Uninverting(_2(7.2.0):C100) Uninverting(_3(7.2.0):C100) Uninverting(_4(7.2.0):C100) Uninverting(_5(7.2.0):C100) Uninverting(_6(7.2.0):C100)))}
[junit4] 2> 293039 INFO (qtp183448557-396) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=334
[junit4] 1> Adding 10 docs with commit=true, numDocs=800
[junit4] 2> 293125 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=68
[junit4] 2> 293126 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817483201871872,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 293126 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817483201871872
[junit4] 2> 293442 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@468a5fd[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 293444 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@468a5fd[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100) Uninverting(_2(7.2.0):C100) Uninverting(_3(7.2.0):C100) Uninverting(_4(7.2.0):C100) Uninverting(_5(7.2.0):C100) Uninverting(_6(7.2.0):C100) Uninverting(_7(7.2.0):C100)))}
[junit4] 2> 293445 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 293445 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=318
[junit4] 1> Adding 10 docs with commit=true, numDocs=900
[junit4] 2> 293469 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=22
[junit4] 2> 293471 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817483563630592,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 293471 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817483563630592
[junit4] 2> 293720 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@76d345f4[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 293723 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@76d345f4[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100) Uninverting(_2(7.2.0):C100) Uninverting(_3(7.2.0):C100) Uninverting(_4(7.2.0):C100) Uninverting(_5(7.2.0):C100) Uninverting(_6(7.2.0):C100) Uninverting(_7(7.2.0):C100) Uninverting(_8(7.2.0):C100)))}
[junit4] 2> 293723 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 293723 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=252
[junit4] 1> Adding 10 docs with commit=true, numDocs=1000
[junit4] 2> 293755 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=29
[junit4] 2> 293800 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817483908612096,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 293800 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817483908612096
[junit4] 2> 294168 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3be019f2[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 294170 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@3be019f2[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100) Uninverting(_2(7.2.0):C100) Uninverting(_3(7.2.0):C100) Uninverting(_4(7.2.0):C100) Uninverting(_5(7.2.0):C100) Uninverting(_6(7.2.0):C100) Uninverting(_7(7.2.0):C100) Uninverting(_8(7.2.0):C100) Uninverting(_9(7.2.0):C100)))}
[junit4] 2> 294170 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 294171 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=370
[junit4] 2> 294173 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&_stateVer_=cdcr-source:4&wt=javabin&version=2} hits=1000 status=0 QTime=0
[junit4] 2> 295013 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 295014 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60334/solr ready
[junit4] 2> 296302 INFO (qtp547630103-352) [n:127.0.0.1:37550_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=cdcr-target&name=cdcr-target&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 296322 INFO (OverseerThreadFactory-135-thread-1-processing-n:127.0.0.1:37550_solr) [n:127.0.0.1:37550_solr ] o.a.s.c.CreateCollectionCmd Create collection cdcr-target
[junit4] 2> 296516 INFO (OverseerStateUpdate-98863591063945219-127.0.0.1:37550_solr-n_0000000000) [n:127.0.0.1:37550_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"cdcr-target",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"cdcr-target_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:37550/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 297028 INFO (qtp547630103-349) [n:127.0.0.1:37550_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=cdcr-target&newCollection=true&name=cdcr-target_shard1_replica_n1&action=CREATE&numShards=1&collection=cdcr-target&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 297029 INFO (qtp547630103-349) [n:127.0.0.1:37550_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 297293 INFO (zkCallback-67-thread-1-processing-n:127.0.0.1:37550_solr) [n:127.0.0.1:37550_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/state.json] for collection [cdcr-target] has occurred - updating... (live nodes size: [1])
[junit4] 2> 297333 INFO (zkCallback-67-thread-2-processing-n:127.0.0.1:37550_solr) [n:127.0.0.1:37550_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/state.json] for collection [cdcr-target] has occurred - updating... (live nodes size: [1])
[junit4] 2> 298214 INFO (qtp547630103-349) [n:127.0.0.1:37550_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 298272 INFO (qtp547630103-349) [n:127.0.0.1:37550_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed sche
[...truncated too long message...]
questHandlerTest, SolrCloudExampleTest, HighlighterConfigTest, DocValuesMultiTest, DeleteLastCustomShardedReplicaTest, TestDelegationWithHadoopAuth, TestCustomStream, TestSearcherReuse, TestConfigSetsAPIZkFailure, TestDefaultStatsCache, TestPerFieldSimilarity, HdfsChaosMonkeySafeLeaderTest, VMParamsZkACLAndCredentialsProvidersTest, SuggesterTSTTest, ClassificationUpdateProcessorFactoryTest, DebugComponentTest, TestDocTermOrds, OpenCloseCoreStressTest, CollectionTooManyReplicasTest, OutputWriterTest, BadIndexSchemaTest, HdfsBasicDistributedZk2Test, TestFastWriter, CloudExitableDirectoryReaderTest, SolrIndexConfigTest, SchemaVersionSpecificBehaviorTest, TestAnalyzedSuggestions, TestExceedMaxTermLength, HdfsUnloadDistributedZkTest, NotRequiredUniqueKeyTest, SystemLogListenerTest, MergeStrategyTest, SpatialRPTFieldTypeTest, TestSolrCoreSnapshots, MBeansHandlerTest, MigrateRouteKeyTest, TestSegmentSorting, TestBackupRepositoryFactory, BinaryUpdateRequestHandlerTest, SharedFSAutoReplicaFailoverTest, TestCollectionAPIs, BufferStoreTest, TestMiniSolrCloudClusterSSL]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestMiniSolrCloudClusterSSL -Dtests.seed=3D43DFAB320A40C4 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es-MX -Dtests.timezone=Pacific/Port_Moresby -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 0.00s J1 | TestMiniSolrCloudClusterSSL (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL:
[junit4] > 1) Thread[id=9145, name=OverseerAutoScalingTriggerThread-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=WAITING, group=Overseer autoscaling triggers]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:161)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 2) Thread[id=9053, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 3) Thread[id=9146, name=ScheduledTrigger-4119-thread-1, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:389)
[junit4] > at org.apache.solr.cloud.ZkDistributedQueue.offer(ZkDistributedQueue.java:293)
[junit4] > at org.apache.solr.cloud.autoscaling.TriggerEventQueue.offerEvent(TriggerEventQueue.java:58)
[junit4] > at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.enqueue(ScheduledTriggers.java:411)
[junit4] > at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$3(ScheduledTriggers.java:205)
[junit4] > at org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$237/524708820.process(Unknown Source)
[junit4] > at org.apache.solr.cloud.autoscaling.NodeLostTrigger.run(NodeLostTrigger.java:163)
[junit4] > at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.run(ScheduledTriggers.java:453)
[junit4] > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] > at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 4) Thread[id=9076, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=9143, name=OverseerStateUpdate-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=TIMED_WAITING, group=Overseer state updater.]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:435)
[junit4] > at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:202)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=9144, name=OverseerCollectionConfigSetProcessor-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=TIMED_WAITING, group=Overseer collection creation process.]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
[junit4] > at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387)
[junit4] > at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 7) Thread[id=9108, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-EventThread, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
[junit4] > 8) Thread[id=9054, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 9) Thread[id=9107, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-SendThread(127.0.0.1:48343), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
[junit4] > at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=9146, name=ScheduledTrigger-4119-thread-1, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 2) Thread[id=9107, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-SendThread(127.0.0.1:48343), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
[junit4] > at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4]:0)
[junit4] Completed [310/744 (3!)] on J1 in 255.65s, 4 tests, 1 failure, 2 errors <<< FAILURES!
[...truncated 48579 lines...]