You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2017/12/21 21:43:17 UTC
[JENKINS] Lucene-Solr-Tests-7.2 - Build # 19 - Still Unstable
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.2/19/
3 tests failed.
FAILED: org.apache.solr.cloud.BasicDistributedZk2Test.test
Error Message:
KeeperErrorCode = Session expired for /clusterstate.json
Stack Trace:
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
at __randomizedtesting.SeedInfo.seed([59F090994ABB744:8DCB36D33A57DABC]:0)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:339)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:339)
at org.apache.solr.common.cloud.ZkStateReader.refreshLegacyClusterState(ZkStateReader.java:542)
at org.apache.solr.common.cloud.ZkStateReader.forceUpdateCollection(ZkStateReader.java:352)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.updateMappingsFromZk(AbstractFullDistribZkTestBase.java:673)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.updateMappingsFromZk(AbstractFullDistribZkTestBase.java:668)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:463)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:333)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.RecoveryZkTest.test
Error Message:
Stack Trace:
java.util.concurrent.TimeoutException
at __randomizedtesting.SeedInfo.seed([59F090994ABB744:8DCB36D33A57DABC]:0)
at org.apache.solr.common.cloud.ZkStateReader.waitForState(ZkStateReader.java:1276)
at org.apache.solr.client.solrj.impl.CloudSolrClient.waitForState(CloudSolrClient.java:448)
at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:122)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.security.TestPKIAuthenticationPlugin.test
Error Message:
Stack Trace:
java.lang.NullPointerException
at __randomizedtesting.SeedInfo.seed([59F090994ABB744:8DCB36D33A57DABC]:0)
at org.apache.solr.security.TestPKIAuthenticationPlugin.test(TestPKIAuthenticationPlugin.java:102)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12163 lines...]
[junit4] Suite: org.apache.solr.cloud.RecoveryZkTest
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryZkTest_59F090994ABB744-001/init-core-data-001
[junit4] 2> 1623084 WARN (SUITE-RecoveryZkTest-seed#[59F090994ABB744]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 1623084 INFO (SUITE-RecoveryZkTest-seed#[59F090994ABB744]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 1623086 INFO (SUITE-RecoveryZkTest-seed#[59F090994ABB744]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 1623086 INFO (SUITE-RecoveryZkTest-seed#[59F090994ABB744]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 1623087 INFO (SUITE-RecoveryZkTest-seed#[59F090994ABB744]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryZkTest_59F090994ABB744-001/tempDir-001
[junit4] 2> 1623087 INFO (SUITE-RecoveryZkTest-seed#[59F090994ABB744]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1623105 INFO (Thread-1004) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1623105 INFO (Thread-1004) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1623137 ERROR (Thread-1004) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1623213 INFO (SUITE-RecoveryZkTest-seed#[59F090994ABB744]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:40818
[junit4] 2> 1623250 INFO (zkConnectionManagerCallback-705-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1623273 INFO (jetty-launcher-702-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1623293 INFO (jetty-launcher-702-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1623399 INFO (jetty-launcher-702-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@44b9e289{/solr,null,AVAILABLE}
[junit4] 2> 1623413 INFO (jetty-launcher-702-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3633b927{/solr,null,AVAILABLE}
[junit4] 2> 1623413 INFO (jetty-launcher-702-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@780223be{SSL,[ssl, http/1.1]}{127.0.0.1:34184}
[junit4] 2> 1623413 INFO (jetty-launcher-702-thread-2) [ ] o.e.j.s.Server Started @1647668ms
[junit4] 2> 1623413 INFO (jetty-launcher-702-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34184}
[junit4] 2> 1623413 INFO (jetty-launcher-702-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@349ca5df{SSL,[ssl, http/1.1]}{127.0.0.1:37634}
[junit4] 2> 1623413 INFO (jetty-launcher-702-thread-1) [ ] o.e.j.s.Server Started @1647668ms
[junit4] 2> 1623413 INFO (jetty-launcher-702-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37634}
[junit4] 2> 1623414 ERROR (jetty-launcher-702-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1623414 INFO (jetty-launcher-702-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 1623414 INFO (jetty-launcher-702-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1623414 INFO (jetty-launcher-702-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1623414 INFO (jetty-launcher-702-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-21T17:58:32.967Z
[junit4] 2> 1623416 ERROR (jetty-launcher-702-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1623416 INFO (jetty-launcher-702-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 1623416 INFO (jetty-launcher-702-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1623416 INFO (jetty-launcher-702-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1623416 INFO (jetty-launcher-702-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-21T17:58:32.969Z
[junit4] 2> 1623495 INFO (zkConnectionManagerCallback-707-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1623502 INFO (jetty-launcher-702-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1623580 INFO (zkConnectionManagerCallback-709-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1623671 INFO (jetty-launcher-702-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1623699 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x1607a3a0be10001, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1623722 INFO (jetty-launcher-702-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40818/solr
[junit4] 2> 1623735 INFO (zkConnectionManagerCallback-713-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1623760 INFO (jetty-launcher-702-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40818/solr
[junit4] 2> 1623809 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x1607a3a0be10003, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1623844 INFO (zkConnectionManagerCallback-717-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1624015 INFO (zkConnectionManagerCallback-721-thread-1-processing-n:127.0.0.1:34184_solr) [n:127.0.0.1:34184_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1624080 INFO (zkConnectionManagerCallback-719-thread-1-processing-n:127.0.0.1:37634_solr) [n:127.0.0.1:37634_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1624413 INFO (jetty-launcher-702-thread-1) [n:127.0.0.1:37634_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1624413 INFO (jetty-launcher-702-thread-1) [n:127.0.0.1:37634_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37634_solr
[junit4] 2> 1624426 INFO (jetty-launcher-702-thread-1) [n:127.0.0.1:37634_solr ] o.a.s.c.Overseer Overseer (id=99213581528137733-127.0.0.1:37634_solr-n_0000000000) starting
[junit4] 2> 1624520 INFO (jetty-launcher-702-thread-1) [n:127.0.0.1:37634_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37634_solr
[junit4] 2> 1624594 INFO (zkCallback-718-thread-1-processing-n:127.0.0.1:37634_solr) [n:127.0.0.1:37634_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1624758 INFO (jetty-launcher-702-thread-2) [n:127.0.0.1:34184_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1624759 INFO (jetty-launcher-702-thread-2) [n:127.0.0.1:34184_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1624796 INFO (jetty-launcher-702-thread-2) [n:127.0.0.1:34184_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34184_solr
[junit4] 2> 1624829 INFO (zkCallback-718-thread-1-processing-n:127.0.0.1:37634_solr) [n:127.0.0.1:37634_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1624922 INFO (zkCallback-720-thread-1-processing-n:127.0.0.1:34184_solr) [n:127.0.0.1:34184_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1625087 INFO (jetty-launcher-702-thread-2) [n:127.0.0.1:34184_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34184.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5a179470
[junit4] 2> 1625124 INFO (jetty-launcher-702-thread-2) [n:127.0.0.1:34184_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34184.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5a179470
[junit4] 2> 1625124 INFO (jetty-launcher-702-thread-2) [n:127.0.0.1:34184_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34184.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5a179470
[junit4] 2> 1625125 INFO (jetty-launcher-702-thread-2) [n:127.0.0.1:34184_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryZkTest_59F090994ABB744-001/tempDir-001/node2/.
[junit4] 2> 1625330 INFO (jetty-launcher-702-thread-1) [n:127.0.0.1:37634_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37634.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5a179470
[junit4] 2> 1625383 INFO (jetty-launcher-702-thread-1) [n:127.0.0.1:37634_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37634.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5a179470
[junit4] 2> 1625384 INFO (jetty-launcher-702-thread-1) [n:127.0.0.1:37634_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37634.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5a179470
[junit4] 2> 1625385 INFO (jetty-launcher-702-thread-1) [n:127.0.0.1:37634_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryZkTest_59F090994ABB744-001/tempDir-001/node1/.
[junit4] 2> 1625643 INFO (zkConnectionManagerCallback-726-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1625697 INFO (zkConnectionManagerCallback-730-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1625698 INFO (SUITE-RecoveryZkTest-seed#[59F090994ABB744]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1625699 INFO (SUITE-RecoveryZkTest-seed#[59F090994ABB744]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40818/solr ready
[junit4] 2> 1625842 INFO (TEST-RecoveryZkTest.test-seed#[59F090994ABB744]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 1626087 INFO (qtp1747717070-3537) [n:127.0.0.1:34184_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1626171 INFO (OverseerThreadFactory-1409-thread-1-processing-n:127.0.0.1:37634_solr) [n:127.0.0.1:37634_solr ] o.a.s.c.CreateCollectionCmd Create collection recoverytest
[junit4] 2> 1626284 INFO (OverseerStateUpdate-99213581528137733-127.0.0.1:37634_solr-n_0000000000) [n:127.0.0.1:37634_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"recoverytest",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"recoverytest_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:37634/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 1626320 INFO (OverseerStateUpdate-99213581528137733-127.0.0.1:37634_solr-n_0000000000) [n:127.0.0.1:37634_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"recoverytest",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"recoverytest_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:34184/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 1626865 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1626865 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1627138 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1627139 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1627352 INFO (zkCallback-718-thread-1-processing-n:127.0.0.1:37634_solr) [n:127.0.0.1:37634_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1627359 INFO (zkCallback-720-thread-1-processing-n:127.0.0.1:34184_solr) [n:127.0.0.1:34184_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1627360 INFO (zkCallback-718-thread-2-processing-n:127.0.0.1:37634_solr) [n:127.0.0.1:37634_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1627360 INFO (zkCallback-720-thread-2-processing-n:127.0.0.1:34184_solr) [n:127.0.0.1:34184_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1628297 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 1628376 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.s.IndexSchema [recoverytest_shard1_replica_n2] Schema name=minimal
[junit4] 2> 1628379 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1628379 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica_n2' using configuration from collection recoverytest, trusted=true
[junit4] 2> 1628392 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 1628412 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34184.solr.core.recoverytest.shard1.replica_n2' (registry 'solr.core.recoverytest.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5a179470
[junit4] 2> 1628412 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1628412 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.SolrCore [[recoverytest_shard1_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryZkTest_59F090994ABB744-001/tempDir-001/node2/recoverytest_shard1_replica_n2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryZkTest_59F090994ABB744-001/tempDir-001/node2/./recoverytest_shard1_replica_n2/data/]
[junit4] 2> 1628502 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.s.IndexSchema [recoverytest_shard1_replica_n1] Schema name=minimal
[junit4] 2> 1628504 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1628504 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica_n1' using configuration from collection recoverytest, trusted=true
[junit4] 2> 1628504 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37634.solr.core.recoverytest.shard1.replica_n1' (registry 'solr.core.recoverytest.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5a179470
[junit4] 2> 1628504 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1628505 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.SolrCore [[recoverytest_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryZkTest_59F090994ABB744-001/tempDir-001/node1/recoverytest_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryZkTest_59F090994ABB744-001/tempDir-001/node1/./recoverytest_shard1_replica_n1/data/]
[junit4] 2> 1629134 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1629134 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1629135 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1629148 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1629149 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1a0d1f27[recoverytest_shard1_replica_n1] main]
[junit4] 2> 1629151 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 1629151 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 1629168 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1629205 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1629205 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1629206 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1629206 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1629208 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@473dba15[recoverytest_shard1_replica_n2] main]
[junit4] 2> 1629221 INFO (searcherExecutor-1415-thread-1-processing-n:127.0.0.1:37634_solr x:recoverytest_shard1_replica_n1 s:shard1 c:recoverytest r:core_node3) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.SolrCore [recoverytest_shard1_replica_n1] Registered new searcher Searcher@1a0d1f27[recoverytest_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1629221 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1587417310847565824
[junit4] 2> 1629241 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 1629242 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 1629242 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1629243 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1587417310869585920
[junit4] 2> 1629260 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 1629278 INFO (searcherExecutor-1414-thread-1-processing-n:127.0.0.1:34184_solr x:recoverytest_shard1_replica_n2 s:shard1 c:recoverytest r:core_node4) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.SolrCore [recoverytest_shard1_replica_n2] Registered new searcher Searcher@473dba15[recoverytest_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1629468 INFO (zkCallback-720-thread-2-processing-n:127.0.0.1:34184_solr) [n:127.0.0.1:34184_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1629469 INFO (zkCallback-720-thread-1-processing-n:127.0.0.1:34184_solr) [n:127.0.0.1:34184_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1629469 INFO (zkCallback-718-thread-2-processing-n:127.0.0.1:37634_solr) [n:127.0.0.1:37634_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1629469 INFO (zkCallback-718-thread-1-processing-n:127.0.0.1:37634_solr) [n:127.0.0.1:37634_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1629774 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1629774 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1629774 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:34184/solr/recoverytest_shard1_replica_n2/
[junit4] 2> 1629774 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica_n2 url=https://127.0.0.1:34184/solr START replicas=[https://127.0.0.1:37634/solr/recoverytest_shard1_replica_n1/] nUpdates=100
[junit4] 2> 1629865 INFO (qtp754565780-3525) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.S.Request [recoverytest_shard1_replica_n1] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1630103 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica_n2 url=https://127.0.0.1:34184/solr DONE. We have no versions. sync failed.
[junit4] 2> 1630103 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 1630103 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 1630103 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1630106 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:34184/solr/recoverytest_shard1_replica_n2/ shard1
[junit4] 2> 1630351 INFO (zkCallback-720-thread-1-processing-n:127.0.0.1:34184_solr) [n:127.0.0.1:34184_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1630351 INFO (zkCallback-720-thread-2-processing-n:127.0.0.1:34184_solr) [n:127.0.0.1:34184_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1630352 INFO (zkCallback-718-thread-3-processing-n:127.0.0.1:37634_solr) [n:127.0.0.1:37634_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1630355 INFO (zkCallback-718-thread-1-processing-n:127.0.0.1:37634_solr) [n:127.0.0.1:37634_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1630376 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1630378 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3513
[junit4] 2> 1630397 INFO (qtp754565780-3530) [n:127.0.0.1:37634_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3259
[junit4] 2> 1630478 INFO (qtp1747717070-3537) [n:127.0.0.1:34184_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1630603 INFO (zkCallback-720-thread-2-processing-n:127.0.0.1:34184_solr) [n:127.0.0.1:34184_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1630603 INFO (zkCallback-720-thread-1-processing-n:127.0.0.1:34184_solr) [n:127.0.0.1:34184_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1630609 INFO (zkCallback-718-thread-1-processing-n:127.0.0.1:37634_solr) [n:127.0.0.1:37634_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1630609 INFO (zkCallback-718-thread-2-processing-n:127.0.0.1:37634_solr) [n:127.0.0.1:37634_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1631485 INFO (qtp1747717070-3537) [n:127.0.0.1:34184_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=5398
[junit4] 2> 1631584 INFO (TEST-RecoveryZkTest.test-seed#[59F090994ABB744]) [ ] o.a.s.c.RecoveryZkTest Indexing 700 documents
[junit4] 2> 1632134 INFO (TEST-RecoveryZkTest.test-seed#[59F090994ABB744]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@349ca5df{SSL,[ssl, http/1.1]}{127.0.0.1:0}
[junit4] 2> 1632133 ERROR (updateExecutor-714-thread-2-processing-https:////127.0.0.1:37634//solr//recoverytest_shard1_replica_n1 x:recoverytest_shard1_replica_n2 r:core_node4 n:127.0.0.1:34184_solr s:shard1 c:recoverytest) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.ErrorReportingConcurrentUpdateSolrClient error
[junit4] 2> javax.net.ssl.SSLHandshakeException: Remote host closed connection during handshake
[junit4] 2> at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:1002)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1385)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1413)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1397)
[junit4] 2> at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:396)
[junit4] 2> at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:355)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:142)
[junit4] 2> at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:359)
[junit4] 2> at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:381)
[junit4] 2> at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:237)
[junit4] 2> at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
[junit4] 2> at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
[junit4] 2> at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
[junit4] 2> at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:347)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:182)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.io.EOFException: SSL peer shut down incorrectly
[junit4] 2> at sun.security.ssl.InputRecord.read(InputRecord.java:505)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983)
[junit4] 2> ... 22 more
[junit4] 2> 1632146 INFO (TEST-RecoveryZkTest.test-seed#[59F090994ABB744]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1040643164
[junit4] 2> 1632161 INFO (TEST-RecoveryZkTest.test-seed#[59F090994ABB744]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 1632162 INFO (TEST-RecoveryZkTest.test-seed#[59F090994ABB744]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@40f7b247: rootName = solr_37634, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@78a62b41
[junit4] 2> 1632222 WARN (qtp1747717070-3533) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.DistributedUpdateProcessor Error sending update to https://127.0.0.1:37634/solr
[junit4] 2> javax.net.ssl.SSLHandshakeException: Remote host closed connection during handshake
[junit4] 2> at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:1002)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1385)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1413)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1397)
[junit4] 2> at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:396)
[junit4] 2> at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:355)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:142)
[junit4] 2> at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:359)
[junit4] 2> at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:381)
[junit4] 2> at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:237)
[junit4] 2> at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
[junit4] 2> at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
[junit4] 2> at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
[junit4] 2> at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:347)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:182)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.io.EOFException: SSL peer shut down incorrectly
[junit4] 2> at sun.security.ssl.InputRecord.read(InputRecord.java:505)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983)
[junit4] 2> ... 22 more
[junit4] 2> 1632223 ERROR (qtp1747717070-3533) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on replica https://127.0.0.1:37634/solr/recoverytest_shard1_replica_n1/
[junit4] 2> java.io.EOFException: SSL peer shut down incorrectly
[junit4] 2> at sun.security.ssl.InputRecord.read(InputRecord.java:505)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1385)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1413)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1397)
[junit4] 2> at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:396)
[junit4] 2> at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:355)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:142)
[junit4] 2> at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:359)
[junit4] 2> at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:381)
[junit4] 2> at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:237)
[junit4] 2> at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
[junit4] 2> at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
[junit4] 2> at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
[junit4] 2> at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:347)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:182)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1632236 INFO (TEST-RecoveryZkTest.test-seed#[59F090994ABB744]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 1632236 INFO (TEST-RecoveryZkTest.test-seed#[59F090994ABB744]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5a6d4453: rootName = solr_37634, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@355a5f2b
[junit4] 2> 1632254 ERROR (updateExecutor-714-thread-1-processing-https:////127.0.0.1:37634//solr//recoverytest_shard1_replica_n1 x:recoverytest_shard1_replica_n2 r:core_node4 n:127.0.0.1:34184_solr s:shard1 c:recoverytest) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.ErrorReportingConcurrentUpdateSolrClient error
[junit4] 2> java.net.SocketException: Broken pipe (Write failed)
[junit4] 2> at java.net.SocketOutputStream.socketWrite0(Native Method)
[junit4] 2> at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:111)
[junit4] 2> at java.net.SocketOutputStream.write(SocketOutputStream.java:155)
[junit4] 2> at sun.security.ssl.OutputRecord.writeBuffer(OutputRecord.java:431)
[junit4] 2> at sun.security.ssl.OutputRecord.write(OutputRecord.java:417)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.writeRecordInternal(SSLSocketImpl.java:886)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.writeRecord(SSLSocketImpl.java:857)
[junit4] 2> at sun.security.ssl.AppOutputStream.write(AppOutputStream.java:123)
[junit4] 2> at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124)
[junit4] 2> at org.apache.http.impl.io.SessionOutputBufferImpl.flushBuffer(SessionOutputBufferImpl.java:136)
[junit4] 2> at org.apache.http.impl.io.SessionOutputBufferImpl.flush(SessionOutputBufferImpl.java:144)
[junit4] 2> at org.apache.http.impl.io.ChunkedOutputStream.close(ChunkedOutputStream.java:203)
[junit4] 2> at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:157)
[junit4] 2> at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:160)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
[junit4] 2> at org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:118)
[junit4] 2> at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
[junit4] 2> at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
[junit4] 2> at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
[junit4] 2> at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
[junit4] 2> at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:347)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:182)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1632258 INFO (TEST-RecoveryZkTest.test-seed#[59F090994ABB744]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 1632258 INFO (TEST-RecoveryZkTest.test-seed#[59F090994ABB744]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@bb3296b: rootName = solr_37634, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@49048b2e
[junit4] 2> 1632259 WARN (qtp1747717070-3539) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.DistributedUpdateProcessor Error sending update to https://127.0.0.1:37634/solr
[junit4] 2> java.net.SocketException: Broken pipe (Write failed)
[junit4] 2> at java.net.SocketOutputStream.socketWrite0(Native Method)
[junit4] 2> at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:111)
[junit4] 2> at java.net.SocketOutputStream.write(SocketOutputStream.java:155)
[junit4] 2> at sun.security.ssl.OutputRecord.writeBuffer(OutputRecord.java:431)
[junit4] 2> at sun.security.ssl.OutputRecord.write(OutputRecord.java:417)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.writeRecordInternal(SSLSocketImpl.java:886)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.writeRecord(SSLSocketImpl.java:857)
[junit4] 2> at sun.security.ssl.AppOutputStream.write(AppOutputStream.java:123)
[junit4] 2> at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124)
[junit4] 2> at org.apache.http.impl.io.SessionOutputBufferImpl.flushBuffer(SessionOutputBufferImpl.java:136)
[junit4] 2> at org.apache.http.impl.io.SessionOutputBufferImpl.flush(SessionOutputBufferImpl.java:144)
[junit4] 2> at org.apache.http.impl.io.ChunkedOutputStream.close(ChunkedOutputStream.java:203)
[junit4] 2> at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:157)
[junit4] 2> at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:160)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
[junit4] 2> at org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:118)
[junit4] 2> at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
[junit4] 2> at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
[junit4] 2> at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
[junit4] 2> at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
[junit4] 2> at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:347)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:182)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1632259 ERROR (qtp1747717070-3539) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on replica https://127.0.0.1:37634/solr/recoverytest_shard1_replica_n1/
[junit4] 2> java.net.SocketException: Broken pipe (Write failed)
[junit4] 2> at java.net.SocketOutputStream.socketWrite0(Native Method)
[junit4] 2> at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:111)
[junit4] 2> at java.net.SocketOutputStream.write(SocketOutputStream.java:155)
[junit4] 2> at sun.security.ssl.OutputRecord.writeBuffer(OutputRecord.java:431)
[junit4] 2> at sun.security.ssl.OutputRecord.write(OutputRecord.java:417)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.writeRecordInternal(SSLSocketImpl.java:886)
[junit4] 2> at sun.security.ssl.SSLSocketImpl.writeRecord(SSLSocketImpl.java:857)
[junit4] 2> at sun.security.ssl.AppOutputStream.write(AppOutputStream.java:123)
[junit4] 2> at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124)
[junit4] 2> at org.apache.http.impl.io.SessionOutputBufferImpl.flushBuffer(SessionOutputBufferImpl.java:136)
[junit4] 2> at org.apache.http.impl.io.SessionOutputBufferImpl.flush(SessionOutputBufferImpl.java:144)
[junit4] 2> at org.apache.http.impl.io.ChunkedOutputStream.close(ChunkedOutputStream.java:203)
[junit4] 2> at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:157)
[junit4] 2> at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:160)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
[junit4] 2> at org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:118)
[junit4] 2> at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
[junit4] 2> at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
[junit4] 2> at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
[junit4] 2> at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
[junit4] 2> at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:347)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:182)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1632261 INFO (qtp1747717070-3533) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.ZkController Put replica core=recoverytest_shard1_replica_n1 coreNodeName=core_node3 on 127.0.0.1:37634_solr into leader-initiated recovery.
[junit4] 2> 1632261 INFO (qtp1747717070-3539) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 (1587417313554989056)]} 0 567
[junit4] 2> 1632261 INFO (qtp1747717070-3533) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 (1587417313554989057)]} 0 458
[junit4] 2> 1632272 ERROR (updateExecutor-714-thread-2-processing-https:////127.0.0.1:37634//solr//recoverytest_shard1_replica_n1 x:recoverytest_shard1_replica_n2 r:core_node4 n:127.0.0.1:34184_solr s:shard1 c:recoverytest) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.ErrorReportingConcurrentUpdateSolrClient error
[junit4] 2> org.apache.http.conn.HttpHostConnectException: Connect to 127.0.0.1:37634 [/127.0.0.1] failed: Connection refused (Connection refused)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:159)
[junit4] 2> at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:359)
[junit4] 2> at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:381)
[junit4] 2> at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:237)
[junit4] 2> at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
[junit4] 2> at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
[junit4] 2> at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
[junit4] 2> at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:347)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:182)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.net.ConnectException: Connection refused (Connection refused)
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:339)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:142)
[junit4] 2> ... 16 more
[junit4] 2> 1632284 INFO (TEST-RecoveryZkTest.test-seed#[59F090994ABB744]) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:37634_solr
[junit4] 2> 1632285 INFO (zkCallback-729-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 1632285 INFO (TEST-RecoveryZkTest.test-seed#[59F090994ABB744]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 1632291 WARN (qtp1747717070-3536) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.DistributedUpdateProcessor Error sending update to https://127.0.0.1:37634/solr
[junit4] 2> org.apache.http.conn.HttpHostConnectException: Connect to 127.0.0.1:37634 [/127.0.0.1] failed: Connection refused (Connection refused)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:159)
[junit4] 2> at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:359)
[junit4] 2> at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:381)
[junit4] 2> at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:237)
[junit4] 2> at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
[junit4] 2> at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
[junit4] 2> at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
[junit4] 2> at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:347)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:182)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.net.ConnectException: Connection refused (Connection refused)
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:339)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:142)
[junit4] 2> ... 16 more
[junit4] 2> 1632291 WARN (qtp1747717070-3536) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.DistributedUpdateProcessor Core core_node4 belonging to recoverytest shard1, does not have error'd node https://127.0.0.1:37634/solr/recoverytest_shard1_replica_n1/ as a replica. No request recovery command will be sent!
[junit4] 2> 1632291 INFO (qtp1747717070-3536) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 (1587417314046771200)]} 0 19
[junit4] 2> 1632292 INFO (OverseerCollectionConfigSetProcessor-99213581528137733-127.0.0.1:37634_solr-n_0000000000) [n:127.0.0.1:37634_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1632293 INFO (qtp1747717070-3540) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 (1587417314067742720)]} 0 0
[junit4] 2> 1632291 INFO (zkCallback-720-thread-1-processing-n:127.0.0.1:34184_solr) [n:127.0.0.1:34184_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 1632294 INFO (qtp1747717070-3540) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 (1587417314069839872)]} 0 0
[junit4] 2> 1632316 INFO (qtp1747717070-3538) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 (1587417314092908544)]} 0 0
[junit4] 2> 1632317 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 (1587417314093957120)]} 0 0
[junit4] 2> 1632319 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 (1587417314096054272)]} 0 0
[junit4] 2> 1632359 INFO (zkCallback-718-thread-3-processing-n:127.0.0.1:37634_solr) [n:127.0.0.1:37634_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 1632364 INFO (qtp1747717070-3540) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-1 (-1587417314104442880)]} 0 69
[junit4] 2> 1632373 INFO (qtp1747717070-3538) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 (1587417314152677376)]} 0 0
[junit4] 2> 1632373 INFO (updateExecutor-714-thread-1-processing-x:recoverytest_shard1_replica_n2 r:core_node4 https:////127.0.0.1:37634//solr//recoverytest_shard1_replica_n1// n:127.0.0.1:34184_solr s:shard1 c:recoverytest) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.LeaderInitiatedRecoveryThread Put replica core=recoverytest_shard1_replica_n1 coreNodeName=core_node3 on 127.0.0.1:37634_solr into leader-initiated recovery.
[junit4] 2> 1632373 WARN (updateExecutor-714-thread-1-processing-x:recoverytest_shard1_replica_n2 r:core_node4 https:////127.0.0.1:37634//solr//recoverytest_shard1_replica_n1// n:127.0.0.1:34184_solr s:shard1 c:recoverytest) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.LeaderInitiatedRecoveryThread Leader is publishing core=recoverytest_shard1_replica_n1 coreNodeName =core_node3 state=down on behalf of un-reachable replica https://127.0.0.1:37634/solr/recoverytest_shard1_replica_n1/
[junit4] 2> 1632374 INFO (qtp1747717070-3539) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-1 (-1587417314153725952)]} 0 0
[junit4] 2> 1632376 INFO (qtp1747717070-3538) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 (1587417314155823104)]} 0 0
[junit4] 2> 1632376 INFO (updateExecutor-714-thread-1-processing-x:recoverytest_shard1_replica_n2 r:core_node4 https:////127.0.0.1:37634//solr//recoverytest_shard1_replica_n1// n:127.0.0.1:34184_solr s:shard1 c:recoverytest) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.LeaderInitiatedRecoveryThread LeaderInitiatedRecoveryThread-recoverytest_shard1_replica_n1 started running to send REQUESTRECOVERY command to https://127.0.0.1:37634/solr/recoverytest_shard1_replica_n1/; will try for a max of 600 secs
[junit4] 2> 1632389 INFO (qtp1747717070-3538) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-5 (1587417314169454592)]} 0 0
[junit4] 2> 1632390 INFO (qtp1747717070-3538) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-6 (1587417314170503168)]} 0 0
[junit4] 2> 1632391 INFO (qtp1747717070-3538) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-2 (-1587417314171551744)]} 0 0
[junit4] 2> 1632391 INFO (qtp1747717070-3538) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-7 (1587417314171551745)]} 0 0
[junit4] 2> 1632392 INFO (qtp1747717070-3538) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-8 (1587417314172600320)]} 0 0
[junit4] 2> 1632555 INFO (updateExecutor-714-thread-1-processing-x:recoverytest_shard1_replica_n2 r:core_node4 https:////127.0.0.1:37634//solr//recoverytest_shard1_replica_n1// n:127.0.0.1:34184_solr s:shard1 c:recoverytest) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.LeaderInitiatedRecoveryThread Asking core=recoverytest_shard1_replica_n1 coreNodeName=core_node3 on https://127.0.0.1:37634/solr to recover
[junit4] 2> 1632586 INFO (qtp1747717070-3540) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-9 (1587417314373926912)]} 0 2
[junit4] 2> 1632587 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-3 (-1587417314377072640)]} 0 0
[junit4] 2> 1632600 INFO (qtp1747717070-3534) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-10 (1587417314390704128)]} 0 12
[junit4] 2> 1632613 ERROR (updateExecutor-714-thread-1-processing-x:recoverytest_shard1_replica_n2 r:core_node4 https:////127.0.0.1:37634//solr//recoverytest_shard1_replica_n1// n:127.0.0.1:34184_solr s:shard1 c:recoverytest) [n:127.0.0.1:34184_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.LeaderInitiatedRecoveryThread https://127.0.0.1:37634/solr: Could not tell a replica to recover:org.apache.solr.client.solrj.SolrServerException: Server refused connection at: https://127.0.0.1:37634/solr
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:650)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
[junit4] 2> at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
[junit4] 2> at org.apache.solr.cloud.LeaderInitiatedRecoveryThread.sendRecoveryCommandWithRetry(LeaderInitiatedRecoveryThread.java:209)
[junit4] 2> at org.apache.solr.cloud.LeaderInitiatedRecoveryThread.run(LeaderInitiatedRecoveryThread.java:95)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.http.conn.HttpHostConnectException: Connect to 127.0.0.1:37634 [/127.0.0.1] failed: Connection refused (Connection refused)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:159)
[junit4] 2> at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:359)
[junit4] 2> at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:381)
[junit4] 2> at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:237)
[junit4] 2> at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
[junit4] 2> at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
[junit4] 2> at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
[junit4] 2> at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
[junit4] 2> ... 10 more
[junit4] 2> Caused by: java.net.ConnectException: Connection refused (Connection refused)
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:339)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(D
[...truncated too long message...]
ache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:333)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J2/temp/solr.cloud.BasicDistributedZk2Test_59F090994ABB744-001
[junit4] 2> Dec 21, 2017 6:33:12 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=21234, maxDocsPerChunk=869, blockSize=612), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=21234, blockSize=612)), sim=RandomSimilarity(queryNorm=true): {}, locale=is-IS, timezone=Europe/Sofia
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=282092704,total=527958016
[junit4] 2> NOTE: All tests run in this JVM: [TestLegacyField, SortSpecParsingTest, DistanceUnitsTest, BlockDirectoryTest, CoreAdminHandlerTest, TestSolrXml, TestSchemaNameResource, HdfsLockFactoryTest, ReturnFieldsTest, TestCloudManagedSchema, HdfsBasicDistributedZkTest, FullSolrCloudDistribCmdsTest, HdfsWriteToMultipleCollectionsTest, TestReplicationHandlerBackup, TestTolerantSearch, WordBreakSolrSpellCheckerTest, TestFiltering, TestSchemaVersionResource, TestFieldCacheVsDocValues, TestConfigSetsAPIZkFailure, DeleteNodeTest, TestMacroExpander, TestAnalyzeInfixSuggestions, TestRealTimeGet, TestDistributedStatsComponentCardinality, LeaderElectionTest, TestSha256AuthenticationProvider, UUIDFieldTest, SolrShardReporterTest, CurrencyFieldTypeTest, SegmentsInfoRequestHandlerTest, TestDistribDocBasedVersion, OverseerStatusTest, SpatialHeatmapFacetsTest, ZkNodePropsTest, SolrJmxReporterTest, TestDFISimilarityFactory, CollectionReloadTest, DefaultValueUpdateProcessorTest, RecoveryZkTest, TestRandomRequestDistribution, TestFieldCacheReopen, TestValueSourceCache, PluginInfoTest, UniqFieldsUpdateProcessorFactoryTest, SharedFSAutoReplicaFailoverTest, MigrateRouteKeyTest, IndexSchemaTest, TestConfigSets, SearchRateTriggerTest, DirectSolrConnectionTest, TestEmbeddedSolrServerConstructors, InfixSuggestersTest, AnalyticsMergeStrategyTest, TriggerIntegrationTest, ConnectionReuseTest, TestTolerantUpdateProcessorRandomCloud, RecoveryAfterSoftCommitTest, MetricsHandlerTest, TestLeaderInitiatedRecoveryThread, ClusterStateTest, SortByFunctionTest, TestFreeTextSuggestions, TestExactSharedStatsCache, TestManagedSchemaThreadSafety, SuggesterWFSTTest, TestHalfAndHalfDocValues, SolrGangliaReporterTest, TestLRUCache, TestDistributedGrouping, TestLFUCache, TestFieldResource, TestDistributedMissingSort, DisMaxRequestHandlerTest, TestSolrCoreProperties, HdfsCollectionsAPIDistributedZkTest, TestConfigOverlay, LeaderElectionContextKeyTest, BlockJoinFacetDistribTest, TestSearchPerf, TestScoreJoinQPNoScore, TestJsonFacetRefinement, HdfsRecoveryZkTest, TestManagedStopFilterFactory, RuleEngineTest, DocValuesNotIndexedTest, TestFunctionQuery, FieldMutatingUpdateProcessorTest, SolrCoreCheckLockOnStartupTest, DistributedDebugComponentTest, TestExtendedDismaxParser, OverseerCollectionConfigSetProcessorTest, SimplePostToolTest, DistributedFacetPivotSmallTest, SystemInfoHandlerTest, TestIndexSearcher, ResponseHeaderTest, TestCollectionsAPIViaSolrCloudCluster, ReplicationFactorTest, TestGroupingSearch, DistributedQueryComponentOptimizationTest, HdfsThreadLeakTest, TestSkipOverseerOperations, SolrMetricManagerTest, BasicDistributedZk2Test]
[junit4] Completed [330/751 (2!)] on J2 in 45.28s, 1 test, 1 error <<< FAILURES!
[...truncated 32 lines...]
[junit4] Suite: org.apache.solr.security.TestPKIAuthenticationPlugin
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J0/temp/solr.security.TestPKIAuthenticationPlugin_59F090994ABB744-001/init-core-data-001
[junit4] 2> 3873300 WARN (SUITE-TestPKIAuthenticationPlugin-seed#[59F090994ABB744]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=27 numCloses=27
[junit4] 2> 3873301 INFO (SUITE-TestPKIAuthenticationPlugin-seed#[59F090994ABB744]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 3873315 INFO (SUITE-TestPKIAuthenticationPlugin-seed#[59F090994ABB744]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 3873315 INFO (SUITE-TestPKIAuthenticationPlugin-seed#[59F090994ABB744]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 3873334 INFO (TEST-TestPKIAuthenticationPlugin.test-seed#[59F090994ABB744]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 3886321 ERROR (TEST-TestPKIAuthenticationPlugin.test-seed#[59F090994ABB744]) [ ] o.a.s.s.PKIAuthenticationPlugin Invalid key request timestamp: 1513881360451 , received timestamp: 1513881373078 , TTL: 10000
[junit4] 2> 3886321 INFO (TEST-TestPKIAuthenticationPlugin.test-seed#[59F090994ABB744]) [ ] o.a.s.SolrTestCaseJ4 ###Ending test
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestPKIAuthenticationPlugin -Dtests.method=test -Dtests.seed=59F090994ABB744 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=fr-CH -Dtests.timezone=America/Los_Angeles -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 13.0s J0 | TestPKIAuthenticationPlugin.test <<<
[junit4] > Throwable #1: java.lang.NullPointerException
[junit4] > at __randomizedtesting.SeedInfo.seed([59F090994ABB744:8DCB36D33A57DABC]:0)
[junit4] > at org.apache.solr.security.TestPKIAuthenticationPlugin.test(TestPKIAuthenticationPlugin.java:102)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J0/temp/solr.security.TestPKIAuthenticationPlugin_59F090994ABB744-001
[junit4] 2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=true): {}, locale=fr-CH, timezone=America/Los_Angeles
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=169000968,total=512753664
[junit4] 2> NOTE: All tests run in this JVM: [TestCorePropertiesReload, LeaderElectionIntegrationTest, FastVectorHighlighterTest, TestCloudPseudoReturnFields, TestConfigSetsAPI, TestPushWriter, TestExpandComponent, NoCacheHeaderTest, TestDFRSimilarityFactory, RAMDirectoryFactoryTest, TestHighlightDedupGrouping, DistributedQueryElevationComponentTest, TestLegacyFieldReuse, RestartWhileUpdatingTest, TestHighFrequencyDictionaryFactory, SparseHLLTest, TestSortByMinMaxFunction, AutoAddReplicasPlanActionTest, FullHLLTest, TestScoreJoinQPScore, TestCodecSupport, TestCloudNestedDocsSort, TestTlogReplica, MultiTermTest, TestSuggestSpellingConverter, DirectoryFactoryTest, TestWordDelimiterFilterFactory, TestCrossCoreJoin, TestReRankQParserPlugin, TestReqParamsAPI, MetricsConfigTest, TestDistributedSearch, TestIntervalFaceting, TestSolrJ, TestUseDocValuesAsStored, TestBinaryResponseWriter, TestManagedSynonymFilterFactory, BooleanFieldTest, DocValuesTest, DistributedFacetPivotWhiteBoxTest, PrimitiveFieldTypeTest, TestFieldTypeCollectionResource, ManagedSchemaRoundRobinCloudTest, SpatialRPTFieldTypeTest, TestComplexPhraseQParserPlugin, TestManagedSchema, MissingSegmentRecoveryTest, IndexSchemaRuntimeFieldTest, AnalysisErrorHandlingTest, SolrXmlInZkTest, SearchHandlerTest, SliceStateTest, TestSubQueryTransformer, TestStandardQParsers, TestShortCircuitedRequests, NumberUtilsTest, AnalysisAfterCoreReloadTest, SimpleMLTQParserTest, QueryParsingTest, TestLuceneIndexBackCompat, TestSolrIndexConfig, ForceLeaderTest, DateMathParserTest, CoreMergeIndexesAdminHandlerTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestJsonFacets, RemoteQueryErrorTest, TestXmlQParser, DateRangeFieldTest, CleanupOldIndexTest, TestImplicitCoreProperties, URLClassifyProcessorTest, UpdateParamsTest, TestLocalFSCloudBackupRestore, SimpleFacetsTest, TestSerializedLuceneMatchVersion, TestNumericTokenStream, TestMacros, TestPostingsSolrHighlighter, TestRandomFaceting, TestConfigSetImmutable, SaslZkACLProviderTest, SyncSliceTest, TestOverriddenPrefixQueryForCustomFieldType, SolrInfoBeanTest, DistribCursorPagingTest, TestMergePolicyConfig, SpellCheckCollatorTest, CollectionStateFormat2Test, TestAddFieldRealTimeGet, TestRequestStatusCollectionAPI, ReplicaListTransformerTest, TestSchemaResource, TestIBSimilarityFactory, MergeStrategyTest, SolrPluginUtilsTest, CdcrReplicationDistributedZkTest, TestFieldCacheWithThreads, DateFieldTest, HdfsChaosMonkeyNothingIsSafeTest, SolrSlf4jReporterTest, TestComponentsName, TestConfig, TestSQLHandlerNonCloud, TestManagedResource, SolrGraphiteReporterTest, TestCloudInspectUtil, TestNRTOpen, VMParamsZkACLAndCredentialsProvidersTest, CursorPagingTest, TestCSVLoader, TestNonDefinedSimilarityFactory, LeaderFailoverAfterPartitionTest, SuggestComponentContextFilterQueryTest, SolrCoreTest, BigEndianAscendingWordSerializerTest, SpellCheckComponentTest, TestPKIAuthenticationPlugin]
[junit4] Completed [340/751 (3!)] on J0 in 13.22s, 1 test, 1 error <<< FAILURES!
[...truncated 48763 lines...]
[JENKINS] Lucene-Solr-Tests-7.2 - Build # 20 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.2/20/
9 tests failed.
FAILED: org.apache.solr.cloud.BasicDistributedZkTest.test
Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:49908/v_
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:49908/v_
at __randomizedtesting.SeedInfo.seed([B788DB6DC32A0442:3FDCE4B76DD669BA]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.cloud.BasicDistributedZkTest.testANewCollectionInOneInstanceWithManualShardAssignement(BasicDistributedZkTest.java:801)
at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:374)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
... 52 more
FAILED: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test
Error Message:
shard2 is not consistent. Got 73 from http://127.0.0.1:44524/collection1_shard2_replica_n61 (previous client) and got 71 from http://127.0.0.1:41856/collection1_shard2_replica_n67
Stack Trace:
java.lang.AssertionError: shard2 is not consistent. Got 73 from http://127.0.0.1:44524/collection1_shard2_replica_n61 (previous client) and got 71 from http://127.0.0.1:41856/collection1_shard2_replica_n67
at __randomizedtesting.SeedInfo.seed([B788DB6DC32A0442:3FDCE4B76DD669BA]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1322)
at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:234)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
Error Message:
9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 1) Thread[id=21799, name=zkCallback-5545-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=21522, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=21527, name=zkCallback-5545-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=21798, name=zkCallback-5545-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=21524, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B788DB6DC32A0442]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 6) Thread[id=21889, name=zkCallback-5545-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=21839, name=zkCallback-5545-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=21523, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B788DB6DC32A0442]-SendThread(127.0.0.1:34447), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 9) Thread[id=21525, name=zkConnectionManagerCallback-5546-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:
1) Thread[id=21799, name=zkCallback-5545-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=21522, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=21527, name=zkCallback-5545-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=21798, name=zkCallback-5545-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=21524, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B788DB6DC32A0442]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
6) Thread[id=21889, name=zkCallback-5545-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=21839, name=zkCallback-5545-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=21523, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B788DB6DC32A0442]-SendThread(127.0.0.1:34447), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
9) Thread[id=21525, name=zkConnectionManagerCallback-5546-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([B788DB6DC32A0442]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=21799, name=zkCallback-5545-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=21527, name=zkCallback-5545-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=21798, name=zkCallback-5545-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=21889, name=zkCallback-5545-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=21839, name=zkCallback-5545-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=21523, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B788DB6DC32A0442]-SendThread(127.0.0.1:34447), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 7) Thread[id=21525, name=zkConnectionManagerCallback-5546-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=21799, name=zkCallback-5545-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=21527, name=zkCallback-5545-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=21798, name=zkCallback-5545-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=21889, name=zkCallback-5545-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=21839, name=zkCallback-5545-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=21523, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B788DB6DC32A0442]-SendThread(127.0.0.1:34447), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
7) Thread[id=21525, name=zkConnectionManagerCallback-5546-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([B788DB6DC32A0442]:0)
FAILED: org.apache.solr.cloud.DeleteReplicaTest.deleteLiveReplicaTest
Error Message:
Error from server at http://127.0.0.1:35203/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000004
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:35203/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000004
at __randomizedtesting.SeedInfo.seed([B788DB6DC32A0442:1AE86F66DE15AC37]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.cloud.DeleteReplicaTest.deleteLiveReplicaTest(DeleteReplicaTest.java:49)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.DeleteStatusTest.testProcessAndWaitDeletesAsyncIds
Error Message:
Request id was not deleted by processAndWait call expected same:<NOT_FOUND> was not:<RUNNING>
Stack Trace:
java.lang.AssertionError: Request id was not deleted by processAndWait call expected same:<NOT_FOUND> was not:<RUNNING>
at __randomizedtesting.SeedInfo.seed([B788DB6DC32A0442:8C96A143961B0DE5]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotSame(Assert.java:641)
at org.junit.Assert.assertSame(Assert.java:580)
at org.apache.solr.cloud.DeleteStatusTest.testProcessAndWaitDeletesAsyncIds(DeleteStatusTest.java:100)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit
Error Message:
Shard split did not succeed after a previous failed split attempt left sub-shards in construction state
Stack Trace:
java.lang.AssertionError: Shard split did not succeed after a previous failed split attempt left sub-shards in construction state
at __randomizedtesting.SeedInfo.seed([B788DB6DC32A0442:4EC548C2FF5F49C8]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:294)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestHdfsCloudBackupRestore.test
Error Message:
expected:<COMPLETED> but was:<RUNNING>
Stack Trace:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>
at __randomizedtesting.SeedInfo.seed([B788DB6DC32A0442:3FDCE4B76DD669BA]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:287)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:140)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.common.cloud.TestCollectionStateWatchers.testWaitForStateWatcherIsRetainedOnPredicateFailure
Error Message:
Did not see a fully active cluster after 30 seconds
Stack Trace:
java.lang.AssertionError: Did not see a fully active cluster after 30 seconds
at __randomizedtesting.SeedInfo.seed([A5621D4BB3FC9A7A:2D54BF186B537268]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.common.cloud.TestCollectionStateWatchers.testWaitForStateWatcherIsRetainedOnPredicateFailure(TestCollectionStateWatchers.java:250)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12142 lines...]
[junit4] Suite: org.apache.solr.cloud.TestHdfsCloudBackupRestore
[junit4] 2> 2648050 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J2/temp/solr.cloud.TestHdfsCloudBackupRestore_B788DB6DC32A0442-001/init-core-data-001
[junit4] 2> 2648051 WARN (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
[junit4] 2> 2648051 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 2648076 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 2649492 WARN (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 2649572 WARN (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2649612 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2649718 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_52088_hdfs____.82kkbn/webapp
[junit4] 2> 2652613 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:52088
[junit4] 2> 2654340 WARN (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2654345 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2654829 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_40874_datanode____.7egbr4/webapp
[junit4] 2> 2657257 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40874
[junit4] 2> 2659848 WARN (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2659877 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2660239 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_51880_datanode____hl1zaz/webapp
[junit4] 2> 2660865 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J2/temp/solr.cloud.TestHdfsCloudBackupRestore_B788DB6DC32A0442-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J2/temp/solr.cloud.TestHdfsCloudBackupRestore_B788DB6DC32A0442-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:55997) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 2660940 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x38184c8582cdcb: from storage DS-9ff9bed8-1028-4e7d-832a-9450e26d86e2 node DatanodeRegistration(127.0.0.1:60353, datanodeUuid=68a7425b-216b-48ab-9495-a8ee35b956d2, infoPort=46344, infoSecurePort=0, ipcPort=57207, storageInfo=lv=-56;cid=testClusterID;nsid=1882413460;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 2660944 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x38184c8582cdcb: from storage DS-ca9f2206-2b34-4805-8f78-59114703ef18 node DatanodeRegistration(127.0.0.1:60353, datanodeUuid=68a7425b-216b-48ab-9495-a8ee35b956d2, infoPort=46344, infoSecurePort=0, ipcPort=57207, storageInfo=lv=-56;cid=testClusterID;nsid=1882413460;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2663161 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:51880
[junit4] 2> 2664669 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J2/temp/solr.cloud.TestHdfsCloudBackupRestore_B788DB6DC32A0442-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J2/temp/solr.cloud.TestHdfsCloudBackupRestore_B788DB6DC32A0442-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:55997) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 2664836 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x38184d6d806384: from storage DS-5287836d-8fa1-4600-90b8-78d4f541c30f node DatanodeRegistration(127.0.0.1:51841, datanodeUuid=1f295162-388a-4544-81c8-63e790f5cd38, infoPort=60996, infoSecurePort=0, ipcPort=59270, storageInfo=lv=-56;cid=testClusterID;nsid=1882413460;c=0), blocks: 0, hasStaleStorage: true, processing time: 4 msecs
[junit4] 2> 2664844 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x38184d6d806384: from storage DS-79e039dc-99dd-42af-83e4-ddbadbbb2683 node DatanodeRegistration(127.0.0.1:51841, datanodeUuid=1f295162-388a-4544-81c8-63e790f5cd38, infoPort=60996, infoSecurePort=0, ipcPort=59270, storageInfo=lv=-56;cid=testClusterID;nsid=1882413460;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2665331 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J2/temp/solr.cloud.TestHdfsCloudBackupRestore_B788DB6DC32A0442-001/tempDir-002
[junit4] 2> 2665331 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2665360 INFO (Thread-1231) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2665360 INFO (Thread-1231) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2665660 ERROR (Thread-1231) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2665660 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:34714
[junit4] 2> 2667016 INFO (zkConnectionManagerCallback-2640-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2667038 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160dd1187480000, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2667063 INFO (jetty-launcher-2637-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2667041 INFO (jetty-launcher-2637-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2667181 INFO (jetty-launcher-2637-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@ec4a7ef{/solr,null,AVAILABLE}
[junit4] 2> 2667181 INFO (jetty-launcher-2637-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3ccb78cd{HTTP/1.1,[http/1.1]}{127.0.0.1:39159}
[junit4] 2> 2667181 INFO (jetty-launcher-2637-thread-2) [ ] o.e.j.s.Server Started @2700076ms
[junit4] 2> 2667181 INFO (jetty-launcher-2637-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39159}
[junit4] 2> 2667182 ERROR (jetty-launcher-2637-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2667182 INFO (jetty-launcher-2637-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.1
[junit4] 2> 2667182 INFO (jetty-launcher-2637-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2667182 INFO (jetty-launcher-2637-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2667182 INFO (jetty-launcher-2637-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-09T22:36:43.398Z
[junit4] 2> 2667321 INFO (jetty-launcher-2637-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@15256e59{/solr,null,AVAILABLE}
[junit4] 2> 2667322 INFO (jetty-launcher-2637-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5590e44{HTTP/1.1,[http/1.1]}{127.0.0.1:43801}
[junit4] 2> 2667322 INFO (jetty-launcher-2637-thread-1) [ ] o.e.j.s.Server Started @2700217ms
[junit4] 2> 2667322 INFO (jetty-launcher-2637-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43801}
[junit4] 2> 2667322 ERROR (jetty-launcher-2637-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2667322 INFO (jetty-launcher-2637-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.1
[junit4] 2> 2667322 INFO (jetty-launcher-2637-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2667322 INFO (jetty-launcher-2637-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2667322 INFO (jetty-launcher-2637-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-09T22:36:43.538Z
[junit4] 2> 2667732 INFO (zkConnectionManagerCallback-2642-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2667733 INFO (jetty-launcher-2637-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2667754 INFO (jetty-launcher-2637-thread-2) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2a1a67, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2667876 INFO (jetty-launcher-2637-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34714/solr
[junit4] 2> 2667936 INFO (zkConnectionManagerCallback-2644-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2667984 INFO (jetty-launcher-2637-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2668043 INFO (zkConnectionManagerCallback-2648-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2668074 INFO (jetty-launcher-2637-thread-1) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2a1a67, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2668105 INFO (jetty-launcher-2637-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34714/solr
[junit4] 2> 2668224 INFO (zkConnectionManagerCallback-2652-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2668224 INFO (zkConnectionManagerCallback-2654-thread-1-processing-n:127.0.0.1:39159_solr) [n:127.0.0.1:39159_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2668428 INFO (zkConnectionManagerCallback-2656-thread-1-processing-n:127.0.0.1:43801_solr) [n:127.0.0.1:43801_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2668490 INFO (jetty-launcher-2637-thread-1) [n:127.0.0.1:43801_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2668661 INFO (jetty-launcher-2637-thread-1) [n:127.0.0.1:43801_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43801_solr
[junit4] 2> 2668662 INFO (jetty-launcher-2637-thread-1) [n:127.0.0.1:43801_solr ] o.a.s.c.Overseer Overseer (id=99322259155976198-127.0.0.1:43801_solr-n_0000000000) starting
[junit4] 2> 2668679 INFO (jetty-launcher-2637-thread-1) [n:127.0.0.1:43801_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43801_solr
[junit4] 2> 2668688 INFO (OverseerStateUpdate-99322259155976198-127.0.0.1:43801_solr-n_0000000000) [n:127.0.0.1:43801_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2669060 INFO (jetty-launcher-2637-thread-1) [n:127.0.0.1:43801_solr ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = hdfs,class = org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = {name=hdfs, class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = {location=/backup,solr.hdfs.home=hdfs://localhost:55997/solr,solr.hdfs.confdir=}}
[junit4] 2> 2669060 INFO (jetty-launcher-2637-thread-1) [n:127.0.0.1:43801_solr ] o.a.s.c.b.r.BackupRepositoryFactory Default configuration for backup repository is with configuration params {type = repository,name = hdfs,class = org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = {name=hdfs, class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = {location=/backup,solr.hdfs.home=hdfs://localhost:55997/solr,solr.hdfs.confdir=}}
[junit4] 2> 2669572 INFO (jetty-launcher-2637-thread-1) [n:127.0.0.1:43801_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a1a67
[junit4] 2> 2669616 INFO (jetty-launcher-2637-thread-2) [n:127.0.0.1:39159_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2669648 INFO (jetty-launcher-2637-thread-2) [n:127.0.0.1:39159_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2669794 INFO (jetty-launcher-2637-thread-1) [n:127.0.0.1:43801_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a1a67
[junit4] 2> 2669794 INFO (jetty-launcher-2637-thread-1) [n:127.0.0.1:43801_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a1a67
[junit4] 2> 2669800 INFO (jetty-launcher-2637-thread-2) [n:127.0.0.1:39159_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39159_solr
[junit4] 2> 2669847 INFO (zkCallback-2655-thread-1-processing-n:127.0.0.1:43801_solr) [n:127.0.0.1:43801_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2669847 INFO (jetty-launcher-2637-thread-1) [n:127.0.0.1:43801_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J2/temp/solr.cloud.TestHdfsCloudBackupRestore_B788DB6DC32A0442-001/tempDir-002/node1/.
[junit4] 2> 2669921 INFO (zkCallback-2653-thread-1-processing-n:127.0.0.1:39159_solr) [n:127.0.0.1:39159_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2670429 INFO (jetty-launcher-2637-thread-2) [n:127.0.0.1:39159_solr ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = hdfs,class = org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = {name=hdfs, class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = {location=/backup,solr.hdfs.home=hdfs://localhost:55997/solr,solr.hdfs.confdir=}}
[junit4] 2> 2670432 INFO (jetty-launcher-2637-thread-2) [n:127.0.0.1:39159_solr ] o.a.s.c.b.r.BackupRepositoryFactory Default configuration for backup repository is with configuration params {type = repository,name = hdfs,class = org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = {name=hdfs, class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = {location=/backup,solr.hdfs.home=hdfs://localhost:55997/solr,solr.hdfs.confdir=}}
[junit4] 2> 2670824 INFO (jetty-launcher-2637-thread-2) [n:127.0.0.1:39159_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a1a67
[junit4] 2> 2671072 INFO (jetty-launcher-2637-thread-2) [n:127.0.0.1:39159_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a1a67
[junit4] 2> 2671072 INFO (jetty-launcher-2637-thread-2) [n:127.0.0.1:39159_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a1a67
[junit4] 2> 2671074 INFO (jetty-launcher-2637-thread-2) [n:127.0.0.1:39159_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-core/test/J2/temp/solr.cloud.TestHdfsCloudBackupRestore_B788DB6DC32A0442-001/tempDir-002/node2/.
[junit4] 2> 2671444 INFO (zkConnectionManagerCallback-2661-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2671604 INFO (zkConnectionManagerCallback-2665-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2671605 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2671606 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[B788DB6DC32A0442]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34714/solr ready
[junit4] 2> 2671937 INFO (TEST-TestHdfsCloudBackupRestore.test-seed#[B788DB6DC32A0442]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2671973 INFO (qtp1174534722-6209) [n:127.0.0.1:39159_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&router.name=implicit&version=2&pullReplicas=1&shards=shard1,shard2&property.customKey=customValue&maxShardsPerNode=3&router.field=shard_s&name=hdfsbackuprestore&nrtReplicas=1&action=CREATE&tlogReplicas=1&wt=javabin and sendToOCPQueue=true
[junit4] 2> 2671999 INFO (OverseerThreadFactory-1551-thread-1-processing-n:127.0.0.1:43801_solr) [n:127.0.0.1:43801_solr ] o.a.s.c.CreateCollectionCmd Create collection hdfsbackuprestore
[junit4] 2> 2672098 WARN (OverseerThreadFactory-1551-thread-1-processing-n:127.0.0.1:43801_solr) [n:127.0.0.1:43801_solr ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 3 on collection hdfsbackuprestore is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
[junit4] 2> 2672317 INFO (OverseerStateUpdate-99322259155976198-127.0.0.1:43801_solr-n_0000000000) [n:127.0.0.1:43801_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"hdfsbackuprestore",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"hdfsbackuprestore_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:43801/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2672368 INFO (OverseerStateUpdate-99322259155976198-127.0.0.1:43801_solr-n_0000000000) [n:127.0.0.1:43801_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"hdfsbackuprestore",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"hdfsbackuprestore_shard1_replica_t2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:39159/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2672417 INFO (OverseerStateUpdate-99322259155976198-127.0.0.1:43801_solr-n_0000000000) [n:127.0.0.1:43801_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"hdfsbackuprestore",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"hdfsbackuprestore_shard1_replica_p4",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:43801/solr",
[junit4] 2> "type":"PULL",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2672458 INFO (OverseerStateUpdate-99322259155976198-127.0.0.1:43801_solr-n_0000000000) [n:127.0.0.1:43801_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"hdfsbackuprestore",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"hdfsbackuprestore_shard2_replica_n6",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:39159/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2672564 INFO (OverseerStateUpdate-99322259155976198-127.0.0.1:43801_solr-n_0000000000) [n:127.0.0.1:43801_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"hdfsbackuprestore",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"hdfsbackuprestore_shard2_replica_t8",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:43801/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2672574 INFO (OverseerStateUpdate-99322259155976198-127.0.0.1:43801_solr-n_0000000000) [n:127.0.0.1:43801_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"hdfsbackuprestore",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"hdfsbackuprestore_shard2_replica_p10",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:39159/solr",
[junit4] 2> "type":"PULL",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2672894 INFO (qtp1558578325-6217) [n:127.0.0.1:43801_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node3&name=hdfsbackuprestore_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&wt=javabin
[junit4] 2> 2672895 INFO (qtp1558578325-6217) [n:127.0.0.1:43801_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2672916 INFO (qtp1558578325-6218) [n:127.0.0.1:43801_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node11&name=hdfsbackuprestore_shard2_replica_t8&action=CREATE&numShards=2&shard=shard2&wt=javabin
[junit4] 2> 2672917 INFO (qtp1174534722-6207) [n:127.0.0.1:39159_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node5&name=hdfsbackuprestore_shard1_replica_t2&action=CREATE&numShards=2&shard=shard1&wt=javabin
[junit4] 2> 2672918 INFO (qtp1174534722-6207) [n:127.0.0.1:39159_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2672996 INFO (qtp1174534722-6211) [n:127.0.0.1:39159_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node12&name=hdfsbackuprestore_shard2_replica_p10&action=CREATE&numShards=2&shard=shard2&wt=javabin
[junit4] 2> 2672997 INFO (qtp1558578325-6221) [n:127.0.0.1:43801_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node7&name=hdfsbackuprestore_shard1_replica_p4&action=CREATE&numShards=2&shard=shard1&wt=javabin
[junit4] 2> 2672999 INFO (qtp1174534722-6212) [n:127.0.0.1:39159_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node9&name=hdfsbackuprestore_shard2_replica_n6&action=CREATE&numShards=2&shard=shard2&wt=javabin
[junit4] 2> 2673142 INFO (zkCallback-2655-thread-1-processing-n:127.0.0.1:43801_solr) [n:127.0.0.1:43801_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2673143 INFO (zkCallback-2653-thread-1-processing-n:127.0.0.1:39159_solr) [n:127.0.0.1:39159_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2673269 INFO (zkCallback-2653-thread-2-processing-n:127.0.0.1:39159_solr) [n:127.0.0.1:39159_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2673352 INFO (zkCallback-2653-thread-2-processing-n:127.0.0.1:39159_solr) [n:127.0.0.1:39159_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2673352 INFO (zkCallback-2653-thread-1-processing-n:127.0.0.1:39159_solr) [n:127.0.0.1:39159_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2673368 INFO (zkCallback-2655-thread-1-processing-n:127.0.0.1:43801_solr) [n:127.0.0.1:43801_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2673994 INFO (qtp1558578325-6217) [n:127.0.0.1:43801_solr c:hdfsbackuprestore s:shard1 r:core_node3 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.1
[junit4] 2> 2674011 INFO (qtp1558578325-6218) [n:127.0.0.1:43801_solr c:hdfsbackuprestore s:shard2 r:core_node11 x:hdfsbackuprestore_shard2_replica_t8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.1
[junit4] 2> 2674278 INFO (qtp1558578325-6221) [n:127.0.0.1:43801_solr c:hdfsbackuprestore s:shard1 r:core_node7 x:hdfsbackuprestore_shard1_replica_p4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.1
[junit4] 2> 2674359 INFO (qtp1174534722-6212) [n:127.0.0.1:39159_solr c:hdfsbackuprestore s:shard2 r:core_node9 x:hdfsbackuprestore_shard2_replica_n6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.1
[junit4] 2> 2674362 INFO (qtp1558578325-6221) [n:127.0.0.1:43801_solr c:hdfsbackuprestore s:shard1 r:
[...truncated too long message...]
FO (coreCloseExecutor-530-thread-1) [n:127.0.0.1:54974_solr c:testcollection s:shard3 r:core_node6 x:testcollection_shard3_replica_n5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testcollection.shard3.leader, tag=447029509
[junit4] 2> 522478 INFO (coreCloseExecutor-530-thread-2) [n:127.0.0.1:54974_solr c:delayed s:shard1 r:core_node2 x:delayed_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.delayed.shard1.leader, tag=1330499324
[junit4] 2> 522546 INFO (coreCloseExecutor-532-thread-2) [n:127.0.0.1:45093_solr c:testcollection s:shard1 r:core_node2 x:testcollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testcollection.shard1.replica_n1, tag=1894945288
[junit4] 2> 522546 INFO (coreCloseExecutor-532-thread-2) [n:127.0.0.1:45093_solr c:testcollection s:shard1 r:core_node2 x:testcollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@696f484d: rootName = solr_45093, domain = solr.core.testcollection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.testcollection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@7623f6f7
[junit4] 2> 522590 INFO (coreCloseExecutor-531-thread-2) [n:127.0.0.1:58762_solr c:stateformat1 s:shard1 r:core_node2 x:stateformat1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.stateformat1.shard1.leader, tag=8977697
[junit4] 2> 522612 INFO (coreCloseExecutor-532-thread-2) [n:127.0.0.1:45093_solr c:testcollection s:shard1 r:core_node2 x:testcollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testcollection.shard1.leader, tag=1894945288
[junit4] 2> 522612 INFO (coreCloseExecutor-533-thread-1) [n:127.0.0.1:47669_solr c:falsepredicate s:shard1 r:core_node2 x:falsepredicate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.falsepredicate.shard1.leader, tag=34990408
[junit4] 2> 522646 INFO (coreCloseExecutor-532-thread-3) [n:127.0.0.1:45093_solr c:falsepredicate s:shard3 r:core_node6 x:falsepredicate_shard3_replica_n5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.falsepredicate.shard3.replica_n5, tag=723112203
[junit4] 2> 522646 INFO (coreCloseExecutor-532-thread-3) [n:127.0.0.1:45093_solr c:falsepredicate s:shard3 r:core_node6 x:falsepredicate_shard3_replica_n5] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@41ecb586: rootName = solr_45093, domain = solr.core.falsepredicate.shard3.replica_n5, service url = null, agent id = null] for registry solr.core.falsepredicate.shard3.replica_n5 / com.codahale.metrics.MetricRegistry@73697d96
[junit4] 2> 522702 INFO (coreCloseExecutor-531-thread-1) [n:127.0.0.1:58762_solr c:testcollection s:shard2 r:core_node4 x:testcollection_shard2_replica_n3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testcollection.shard2.replica_n3, tag=116782384
[junit4] 2> 522914 INFO (AutoscalingActionExecutor-446-thread-1-processing-n:127.0.0.1:45093_solr) [n:127.0.0.1:45093_solr ] o.a.s.c.ActionThrottle The last action attempt started 7350ms ago.
[junit4] 2> 522930 INFO (coreCloseExecutor-531-thread-1) [n:127.0.0.1:58762_solr c:testcollection s:shard2 r:core_node4 x:testcollection_shard2_replica_n3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2afbbacc: rootName = solr_58762, domain = solr.core.testcollection.shard2.replica_n3, service url = null, agent id = null] for registry solr.core.testcollection.shard2.replica_n3 / com.codahale.metrics.MetricRegistry@8b10c85
[junit4] 2> 522935 INFO (AutoscalingActionExecutor-446-thread-1-processing-n:127.0.0.1:45093_solr) [n:127.0.0.1:45093_solr ] o.a.s.c.a.ExecutePlanAction No operations to execute for event: {
[junit4] 2> "id":"15084bc455016580T3vgfp88w5b996uoieq78oimak",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":1515544581222000000,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[1515544581222000000],
[junit4] 2> "_enqueue_time_":1515544616513000000,
[junit4] 2> "nodeNames":["127.0.0.1:56031_solr"]}}
[junit4] 2> 522999 INFO (coreCloseExecutor-532-thread-1) [n:127.0.0.1:45093_solr c:currentstate s:shard1 r:core_node2 x:currentstate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.currentstate.shard1.leader, tag=1754171940
[junit4] 2> 522999 INFO (coreCloseExecutor-532-thread-3) [n:127.0.0.1:45093_solr c:falsepredicate s:shard3 r:core_node6 x:falsepredicate_shard3_replica_n5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.falsepredicate.shard3.leader, tag=723112203
[junit4] 2> 523023 INFO (coreCloseExecutor-531-thread-1) [n:127.0.0.1:58762_solr c:testcollection s:shard2 r:core_node4 x:testcollection_shard2_replica_n3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testcollection.shard2.leader, tag=116782384
[junit4] 2> 523526 INFO (jetty-closer-129-thread-1) [ ] o.a.s.c.Overseer Overseer (id=99322722687975434-127.0.0.1:45093_solr-n_0000000001) closing
[junit4] 2> 523542 INFO (OverseerStateUpdate-99322722687975434-127.0.0.1:45093_solr-n_0000000001) [n:127.0.0.1:45093_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:45093_solr
[junit4] 2> 523570 WARN (OverseerAutoScalingTriggerThread-99322722687975434-127.0.0.1:45093_solr-n_0000000001) [n:127.0.0.1:45093_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 523591 INFO (jetty-closer-129-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@20086cb5{/solr,null,UNAVAILABLE}
[junit4] 2> 523611 INFO (jetty-closer-129-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@10348523{/solr,null,UNAVAILABLE}
[junit4] 2> 523690 INFO (jetty-closer-129-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@58c5d069{/solr,null,UNAVAILABLE}
[junit4] 2> 525090 INFO (jetty-closer-129-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@e0dd70b{/solr,null,UNAVAILABLE}
[junit4] 2> 525122 ERROR (SUITE-TestCollectionStateWatchers-seed#[A5621D4BB3FC9A7A]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 525123 INFO (SUITE-TestCollectionStateWatchers-seed#[A5621D4BB3FC9A7A]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39319 39319
[junit4] 2> 525374 INFO (Thread-106) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39319 39319
[junit4] 2> 525375 WARN (Thread-106) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/aliases.json
[junit4] 2> 8 /solr/configs/config
[junit4] 2> 7 /solr/security.json
[junit4] 2> 4 /solr/collections/stateformat1/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 76 /solr/collections/falsepredicate/state.json
[junit4] 2> 64 /solr/collections/testcollection/state.json
[junit4] 2> 12 /solr/collections/waitforstate/state.json
[junit4] 2> 11 /solr/collections/delayed/state.json
[junit4] 2> 9 /solr/collections/stateformat1/state.json
[junit4] 2> 8 /solr/clusterprops.json
[junit4] 2> 8 /solr/clusterstate.json
[junit4] 2> 7 /solr/collections/currentstate/state.json
[junit4] 2> 6 /solr/collections/tobedeleted/state.json
[junit4] 2> 2 /solr/overseer_elect/election/99322722687975435-127.0.0.1:56031_solr-n_0000000002
[junit4] 2> 2 /solr/autoscaling.json
[junit4] 2> 2 /solr/overseer_elect/election/99322722687975434-127.0.0.1:45093_solr-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/live_nodes
[junit4] 2> 8 /solr/collections
[junit4] 2> 2 /solr/overseer/queue
[junit4] 2> 2 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2> 2 /solr/overseer/collection-queue-work
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.2/solr/build/solr-solrj/test/J2/temp/solr.common.cloud.TestCollectionStateWatchers_A5621D4BB3FC9A7A-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=547, maxMBSortInHeap=5.095557124251343, sim=RandomSimilarity(queryNorm=true): {}, locale=ja, timezone=Asia/Omsk
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=243108832,total=361234432
[junit4] 2> NOTE: All tests run in this JVM: [TestCoreAdmin, CoalesceEvaluatorTest, TestCloudSolrClientConnections, LessThanEqualToEvaluatorTest, TestJavaBinCodec, TestSuggesterResponse, NormalizeEvaluatorTest, TangentEvaluatorTest, UuidEvaluatorTest, NoOpResponseParserTest, MultiplyEvaluatorTest, SolrExceptionTest, LargeVolumeJettyTest, OperationsTest, ArrayEvaluatorTest, SelectWithEvaluatorsTest, TestJsonRecordReader, PowerEvaluatorTest, RegressionEvaluatorTest, StreamExpressionToExplanationTest, DocumentAnalysisResponseTest, TestCollectionStateWatchers]
[junit4] Completed [50/146 (1!)] on J2 in 144.83s, 9 tests, 1 failure <<< FAILURES!
[...truncated 46731 lines...]