You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/03/26 09:41:53 UTC
[JENKINS] Lucene-Solr-Tests-7.x - Build # 527 - Unstable
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/527/
4 tests failed.
FAILED: org.apache.solr.cloud.OverseerRolesTest.testOverseerRole
Error Message:
Error from server at https://127.0.0.1:34725/solr: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> <title>Error 404 Not Found</title> </head> <body><h2>HTTP ERROR 404</h2> <p>Problem accessing /solr/admin/collections. Reason: <pre> Not Found</pre></p><h3>Caused by:</h3><pre>javax.servlet.UnavailableException: Error processing the request. CoreContainer is either not initialized or shutting down. at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:349) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:331) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:527) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:530) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:289) at org.eclipse.jetty.io.ssl.SslConnection$3.succeeded(SslConnection.java:149) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) </pre> <hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.8.v20171121</a><hr/> </body> </html>
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:34725/solr: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 404 Not Found</title>
</head>
<body><h2>HTTP ERROR 404</h2>
<p>Problem accessing /solr/admin/collections. Reason:
<pre> Not Found</pre></p><h3>Caused by:</h3><pre>javax.servlet.UnavailableException: Error processing the request. CoreContainer is either not initialized or shutting down.
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:349)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:331)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:527)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at org.eclipse.jetty.server.Server.handle(Server.java:530)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:289)
at org.eclipse.jetty.io.ssl.SslConnection$3.succeeded(SslConnection.java:149)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
</pre>
<hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.8.v20171121</a><hr/>
</body>
</html>
at __randomizedtesting.SeedInfo.seed([206EE6161273345E:C1A51B8229C0028F]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.cloud.OverseerRolesTest.testOverseerRole(OverseerRolesTest.java:123)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.OverseerRolesTest
Error Message:
18 threads leaked from SUITE scope at org.apache.solr.cloud.OverseerRolesTest: 1) Thread[id=13538, name=Thread-6169, state=WAITING, group=TGRP-OverseerRolesTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1725) 2) Thread[id=13644, name=zkConnectionManagerCallback-2336-thread-1-EventThread, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 3) Thread[id=13517, name=zkConnectionManagerCallback-2298-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=13643, name=zkConnectionManagerCallback-2336-thread-1-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 5) Thread[id=13650, name=ScheduledTrigger-3164-thread-2, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=13588, name=zkConnectionManagerCallback-2336-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=13510, name=jetty-launcher-2271-thread-3-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230) at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1249) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1173) 8) Thread[id=13651, name=ScheduledTrigger-3164-thread-4, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=13540, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=13647, name=OverseerCollectionConfigSetProcessor-72345431130177545-127.0.0.1:34725_solr-n_0000000005, state=TIMED_WAITING, group=Overseer collection creation process.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:392) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=13585, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=13649, name=ScheduledTrigger-3164-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=13648, name=OverseerAutoScalingTriggerThread-72345431130177545-127.0.0.1:34725_solr-n_0000000005, state=WAITING, group=Overseer autoscaling triggers] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:177) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=13652, name=ScheduledTrigger-3164-thread-3, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=13490, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=13491, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=13511, name=jetty-launcher-2271-thread-3-EventThread, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 18) Thread[id=13639, name=OverseerStateUpdate-72345431130177545-127.0.0.1:34725_solr-n_0000000005, state=TIMED_WAITING, group=Overseer state updater.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:415) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 18 threads leaked from SUITE scope at org.apache.solr.cloud.OverseerRolesTest:
1) Thread[id=13538, name=Thread-6169, state=WAITING, group=TGRP-OverseerRolesTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1725)
2) Thread[id=13644, name=zkConnectionManagerCallback-2336-thread-1-EventThread, state=WAITING, group=TGRP-OverseerRolesTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
3) Thread[id=13517, name=zkConnectionManagerCallback-2298-thread-1, state=WAITING, group=TGRP-OverseerRolesTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=13643, name=zkConnectionManagerCallback-2336-thread-1-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
5) Thread[id=13650, name=ScheduledTrigger-3164-thread-2, state=TIMED_WAITING, group=TGRP-OverseerRolesTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=13588, name=zkConnectionManagerCallback-2336-thread-1, state=WAITING, group=TGRP-OverseerRolesTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=13510, name=jetty-launcher-2271-thread-3-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230)
at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1249)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1173)
8) Thread[id=13651, name=ScheduledTrigger-3164-thread-4, state=WAITING, group=TGRP-OverseerRolesTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=13540, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=13647, name=OverseerCollectionConfigSetProcessor-72345431130177545-127.0.0.1:34725_solr-n_0000000005, state=TIMED_WAITING, group=Overseer collection creation process.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:392)
at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=13585, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=13649, name=ScheduledTrigger-3164-thread-1, state=WAITING, group=TGRP-OverseerRolesTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=13648, name=OverseerAutoScalingTriggerThread-72345431130177545-127.0.0.1:34725_solr-n_0000000005, state=WAITING, group=Overseer autoscaling triggers]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:177)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=13652, name=ScheduledTrigger-3164-thread-3, state=WAITING, group=TGRP-OverseerRolesTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=13490, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
16) Thread[id=13491, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
17) Thread[id=13511, name=jetty-launcher-2271-thread-3-EventThread, state=WAITING, group=TGRP-OverseerRolesTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
18) Thread[id=13639, name=OverseerStateUpdate-72345431130177545-127.0.0.1:34725_solr-n_0000000005, state=TIMED_WAITING, group=Overseer state updater.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:415)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([206EE6161273345E]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.OverseerRolesTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=13538, name=Thread-6169, state=WAITING, group=TGRP-OverseerRolesTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1725) 2) Thread[id=13649, name=ScheduledTrigger-3164-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=13517, name=zkConnectionManagerCallback-2298-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=13643, name=zkConnectionManagerCallback-2336-thread-1-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054) 5) Thread[id=13652, name=ScheduledTrigger-3164-thread-3, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=13650, name=ScheduledTrigger-3164-thread-2, state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=13588, name=zkConnectionManagerCallback-2336-thread-1, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=13510, name=jetty-launcher-2271-thread-3-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 9) Thread[id=13651, name=ScheduledTrigger-3164-thread-4, state=WAITING, group=TGRP-OverseerRolesTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=13538, name=Thread-6169, state=WAITING, group=TGRP-OverseerRolesTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1725)
2) Thread[id=13649, name=ScheduledTrigger-3164-thread-1, state=WAITING, group=TGRP-OverseerRolesTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=13517, name=zkConnectionManagerCallback-2298-thread-1, state=WAITING, group=TGRP-OverseerRolesTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=13643, name=zkConnectionManagerCallback-2336-thread-1-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)
5) Thread[id=13652, name=ScheduledTrigger-3164-thread-3, state=WAITING, group=TGRP-OverseerRolesTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=13650, name=ScheduledTrigger-3164-thread-2, state=TIMED_WAITING, group=TGRP-OverseerRolesTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=13588, name=zkConnectionManagerCallback-2336-thread-1, state=WAITING, group=TGRP-OverseerRolesTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=13510, name=jetty-launcher-2271-thread-3-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
9) Thread[id=13651, name=ScheduledTrigger-3164-thread-4, state=WAITING, group=TGRP-OverseerRolesTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([206EE6161273345E]:0)
FAILED: org.apache.solr.cloud.cdcr.CdcrVersionReplicationTest.testCdcrDocVersions
Error Message:
Error from server at http://127.0.0.1:46847/f_u/ld: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000008
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:46847/f_u/ld: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000008
at __randomizedtesting.SeedInfo.seed([206EE6161273345E:D8F8EDB4E015DB42]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1105)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:885)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:818)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.cloud.cdcr.BaseCdcrDistributedZkTest.startServers(BaseCdcrDistributedZkTest.java:594)
at org.apache.solr.cloud.cdcr.BaseCdcrDistributedZkTest.createSourceCollection(BaseCdcrDistributedZkTest.java:353)
at org.apache.solr.cloud.cdcr.BaseCdcrDistributedZkTest.baseBefore(BaseCdcrDistributedZkTest.java:179)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 1804 lines...]
[junit4] JVM J2: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J2-20180326_065247_4144214870791478663138.sysout
[junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
[junit4] codec: Asserting, pf: LuceneFixedGap, dvf: Memory
[junit4] <<< JVM J2: EOF ----
[...truncated 10572 lines...]
[junit4] Suite: org.apache.solr.cloud.cdcr.CdcrVersionReplicationTest
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/init-core-data-001
[junit4] 2> 371262 WARN (SUITE-CdcrVersionReplicationTest-seed#[206EE6161273345E]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=210 numCloses=210
[junit4] 2> 371305 INFO (SUITE-CdcrVersionReplicationTest-seed#[206EE6161273345E]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 371307 INFO (SUITE-CdcrVersionReplicationTest-seed#[206EE6161273345E]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 371307 INFO (SUITE-CdcrVersionReplicationTest-seed#[206EE6161273345E]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 371307 INFO (SUITE-CdcrVersionReplicationTest-seed#[206EE6161273345E]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /f_u/ld
[junit4] 2> 371344 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 371360 INFO (Thread-279) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 371360 INFO (Thread-279) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 371456 ERROR (Thread-279) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 371515 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.ZkTestServer start zk server on port:43763
[junit4] 2> 371626 INFO (zkConnectionManagerCallback-280-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 371967 INFO (zkConnectionManagerCallback-282-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 371992 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-cdcr.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 371994 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 371995 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 372029 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 372030 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 372031 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 372050 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 372051 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 372052 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 372085 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 372087 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 372089 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testCdcrDocVersions
[junit4] 2> 373442 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T12:27:37-09:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 373447 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 373447 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 373447 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 373448 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5ceab7b1{/f_u/ld,null,AVAILABLE}
[junit4] 2> 373450 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1742164{HTTP/1.1,[http/1.1]}{127.0.0.1:46847}
[junit4] 2> 373450 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.Server Started @373568ms
[junit4] 2> 373450 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard1, hostContext=/f_u/ld, hostPort=46847, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-001/cores}
[junit4] 2> 373450 ERROR (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 373450 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 373450 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.4.0
[junit4] 2> 373471 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 373471 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 373471 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-26T08:23:47.216Z
[junit4] 2> 373564 INFO (zkConnectionManagerCallback-284-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 373585 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 373585 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-001/solr.xml
[junit4] 2> 373655 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 373655 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 373656 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 373692 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43763/solr
[junit4] 2> 373727 INFO (zkConnectionManagerCallback-288-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 373757 INFO (zkConnectionManagerCallback-290-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 374174 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 374175 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46847_f_u%2Fld
[junit4] 2> 374177 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.c.Overseer Overseer (id=72345298786844676-127.0.0.1:46847_f_u%2Fld-n_0000000000) starting
[junit4] 2> 374232 INFO (zkConnectionManagerCallback-295-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 374234 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43763/solr ready
[junit4] 2> 374243 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46847_f_u%2Fld
[junit4] 2> 374454 INFO (zkCallback-294-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 374454 INFO (OverseerStateUpdate-72345298786844676-127.0.0.1:46847_f_u%2Fld-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 374823 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed
[junit4] 2> 374873 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed
[junit4] 2> 374873 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed
[junit4] 2> 374874 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-001/cores
[junit4] 2> 378862 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T12:27:37-09:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 378918 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 378918 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 378919 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 378919 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4ddf1fa4{/f_u/ld,null,AVAILABLE}
[junit4] 2> 378920 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5e4ea662{HTTP/1.1,[http/1.1]}{127.0.0.1:34923}
[junit4] 2> 378920 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.Server Started @379038ms
[junit4] 2> 378920 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard2, hostContext=/f_u/ld, hostPort=34923, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-002/cores}
[junit4] 2> 378920 ERROR (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 378920 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 378920 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.4.0
[junit4] 2> 378920 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 378920 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 378920 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-26T08:23:52.665Z
[junit4] 2> 379121 INFO (zkConnectionManagerCallback-297-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 379256 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 379256 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-002/solr.xml
[junit4] 2> 379329 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 379329 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 379350 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 379402 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43763/solr
[junit4] 2> 379439 INFO (zkConnectionManagerCallback-301-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 379516 INFO (zkConnectionManagerCallback-303-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 379552 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 379586 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 379588 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34923_f_u%2Fld
[junit4] 2> 379606 INFO (zkCallback-294-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 379621 INFO (zkCallback-289-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 379857 INFO (zkCallback-302-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 380631 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed
[junit4] 2> 380683 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed
[junit4] 2> 380683 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed
[junit4] 2> 380685 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-002/cores
[junit4] 2> 380711 INFO (zkConnectionManagerCallback-308-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 380712 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 380713 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:34923_f_u%2Fld ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43763/solr ready
[junit4] 2> 382172 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T12:27:37-09:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 382190 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 382190 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 382190 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 382191 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2a3d776{/f_u/ld,null,AVAILABLE}
[junit4] 2> 382191 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7c51bdd2{HTTP/1.1,[http/1.1]}{127.0.0.1:37717}
[junit4] 2> 382191 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.Server Started @382309ms
[junit4] 2> 382191 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard3, hostContext=/f_u/ld, hostPort=37717, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-003/cores}
[junit4] 2> 382191 ERROR (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 382192 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 382192 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.4.0
[junit4] 2> 382192 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 382192 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 382192 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-26T08:23:55.937Z
[junit4] 2> 382239 INFO (zkConnectionManagerCallback-310-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 382241 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 382241 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-003/solr.xml
[junit4] 2> 382245 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 382245 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 382246 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 382277 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43763/solr
[junit4] 2> 382324 INFO (zkConnectionManagerCallback-314-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 382391 INFO (zkConnectionManagerCallback-316-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 382440 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 382441 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 382456 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37717_f_u%2Fld
[junit4] 2> 382457 INFO (zkCallback-289-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 382457 INFO (zkCallback-302-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 382475 INFO (zkCallback-294-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 382589 INFO (zkCallback-315-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 382609 INFO (zkCallback-307-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 383858 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed
[junit4] 2> 383896 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed
[junit4] 2> 383896 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed
[junit4] 2> 383914 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-003/cores
[junit4] 2> 383938 INFO (zkConnectionManagerCallback-321-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 383951 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 383951 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:37717_f_u%2Fld ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43763/solr ready
[junit4] 2> 385724 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T12:27:37-09:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 385835 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 385835 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 385835 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 385835 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1800e3a5{/f_u/ld,null,AVAILABLE}
[junit4] 2> 385836 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@26d74f3c{HTTP/1.1,[http/1.1]}{127.0.0.1:43383}
[junit4] 2> 385836 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.e.j.s.Server Started @385954ms
[junit4] 2> 385836 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard4, hostContext=/f_u/ld, hostPort=43383, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-004/cores}
[junit4] 2> 385836 ERROR (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 385836 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 385836 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.4.0
[junit4] 2> 385836 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 385836 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 385836 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-26T08:23:59.581Z
[junit4] 2> 385887 INFO (zkConnectionManagerCallback-323-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 385921 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 385921 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-004/solr.xml
[junit4] 2> 385958 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 385958 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 385959 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 386031 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43763/solr
[junit4] 2> 386092 INFO (zkConnectionManagerCallback-327-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 386180 INFO (zkConnectionManagerCallback-329-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 386231 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 386270 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 386272 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43383_f_u%2Fld
[junit4] 2> 386286 INFO (zkCallback-302-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 386302 INFO (zkCallback-294-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 386302 INFO (zkCallback-307-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 386302 INFO (zkCallback-315-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 386302 INFO (zkCallback-289-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 386395 INFO (zkCallback-320-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 386411 INFO (zkCallback-328-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 388487 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed
[junit4] 2> 388545 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed
[junit4] 2> 388545 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1601b4ed
[junit4] 2> 388559 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrVersionReplicationTest_206EE6161273345E-001/jetty-004/cores
[junit4] 2> 388677 INFO (zkConnectionManagerCallback-334-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 388686 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 388687 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [n:127.0.0.1:43383_f_u%2Fld ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43763/solr ready
[junit4] 2> 388920 INFO (zkConnectionManagerCallback-338-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 388935 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 388936 INFO (TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[206EE6161273345E]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43763/solr ready
[junit4] 2> 388966 INFO (qtp1084406563-1210) [n:127.0.0.1:46847_f_u%2Fld ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collecti
[...truncated too long message...]
ad.run(Thread.java:748)
[junit4] > 15) Thread[id=13490, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 16) Thread[id=13491, name=Connection evictor, state=TIMED_WAITING, group=TGRP-OverseerRolesTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 17) Thread[id=13511, name=jetty-launcher-2271-thread-3-EventThread, state=WAITING, group=TGRP-OverseerRolesTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
[junit4] > 18) Thread[id=13639, name=OverseerStateUpdate-72345431130177545-127.0.0.1:34725_solr-n_0000000005, state=TIMED_WAITING, group=Overseer state updater.]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] > at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:415)
[junit4] > at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([206EE6161273345E]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=13538, name=Thread-6169, state=WAITING, group=TGRP-OverseerRolesTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1725)
[junit4] > 2) Thread[id=13649, name=ScheduledTrigger-3164-thread-1, state=WAITING, group=TGRP-OverseerRolesTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 3) Thread[id=13517, name=zkConnectionManagerCallback-2298-thread-1, state=WAITING, group=TGRP-OverseerRolesTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 4) Thread[id=13643, name=zkConnectionManagerCallback-2336-thread-1-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)
[junit4] > 5) Thread[id=13652, name=ScheduledTrigger-3164-thread-3, state=WAITING, group=TGRP-OverseerRolesTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=13650, name=ScheduledTrigger-3164-thread-2, state=TIMED_WAITING, group=TGRP-OverseerRolesTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 7) Thread[id=13588, name=zkConnectionManagerCallback-2336-thread-1, state=WAITING, group=TGRP-OverseerRolesTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 8) Thread[id=13510, name=jetty-launcher-2271-thread-3-SendThread(127.0.0.1:35734), state=TIMED_WAITING, group=TGRP-OverseerRolesTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
[junit4] > 9) Thread[id=13651, name=ScheduledTrigger-3164-thread-4, state=WAITING, group=TGRP-OverseerRolesTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([206EE6161273345E]:0)
[junit4] Completed [328/785 (2!)] on J1 in 121.62s, 1 test, 3 errors <<< FAILURES!
[...truncated 51066 lines...]