You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2019/11/29 21:46:28 UTC
[JENKINS] Lucene-Solr-BadApples-NightlyTests-8.x - Build # 45 -
Still Unstable
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-8.x/45/
3 tests failed.
FAILED: org.apache.solr.cloud.LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud
Error Message:
Error from server at https://127.0.0.1:33724/solr: Underlying core creation failed while creating collection: legacyFalse
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:33724/solr: Underlying core creation failed while creating collection: legacyFalse
at __randomizedtesting.SeedInfo.seed([A2E34C95B48671C0:73E4BE101089FAF2]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:665)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:265)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1127)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:896)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:828)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:228)
at org.apache.solr.cloud.LegacyCloudClusterPropTest.createAndTest(LegacyCloudClusterPropTest.java:95)
at org.apache.solr.cloud.LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud(LegacyCloudClusterPropTest.java:79)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.hdfs.StressHdfsTest.test
Error Message:
Error from server at http://127.0.0.1:37715/_/delete_data_dir: Expected mime type application/octet-stream but got application/json. { "error":{ "metadata":[ "error-class","org.apache.solr.common.SolrException", "root-error-class","java.net.SocketTimeoutException"], "msg":"Error trying to proxy request for url: http://127.0.0.1:42755/_/delete_data_dir/update", "trace":"org.apache.solr.common.SolrException: Error trying to proxy request for url: http://127.0.0.1:42755/_/delete_data_dir/update\n\tat org.apache.solr.servlet.HttpSolrCall.remoteQuery(HttpSolrCall.java:735)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:562)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:419)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1711)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1347)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1678)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1249)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)\n\tat org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:505)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)\n\tat org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: java.net.SocketTimeoutException: Read timed out\n\tat java.net.SocketInputStream.socketRead0(Native Method)\n\tat java.net.SocketInputStream.socketRead(SocketInputStream.java:116)\n\tat java.net.SocketInputStream.read(SocketInputStream.java:171)\n\tat java.net.SocketInputStream.read(SocketInputStream.java:141)\n\tat org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)\n\tat org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)\n\tat org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)\n\tat org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)\n\tat org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)\n\tat org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)\n\tat org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)\n\tat org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)\n\tat org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)\n\tat org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)\n\tat org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:120)\n\tat org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)\n\tat org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)\n\tat org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)\n\tat org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)\n\tat org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)\n\tat org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)\n\tat org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)\n\tat org.apache.solr.servlet.HttpSolrCall.remoteQuery(HttpSolrCall.java:707)\n\t... 30 more\n", "code":500}}
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:37715/_/delete_data_dir: Expected mime type application/octet-stream but got application/json. {
"error":{
"metadata":[
"error-class","org.apache.solr.common.SolrException",
"root-error-class","java.net.SocketTimeoutException"],
"msg":"Error trying to proxy request for url: http://127.0.0.1:42755/_/delete_data_dir/update",
"trace":"org.apache.solr.common.SolrException: Error trying to proxy request for url: http://127.0.0.1:42755/_/delete_data_dir/update\n\tat org.apache.solr.servlet.HttpSolrCall.remoteQuery(HttpSolrCall.java:735)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:562)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:419)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1711)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1347)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1678)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1249)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)\n\tat org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:505)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)\n\tat org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: java.net.SocketTimeoutException: Read timed out\n\tat java.net.SocketInputStream.socketRead0(Native Method)\n\tat java.net.SocketInputStream.socketRead(SocketInputStream.java:116)\n\tat java.net.SocketInputStream.read(SocketInputStream.java:171)\n\tat java.net.SocketInputStream.read(SocketInputStream.java:141)\n\tat org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)\n\tat org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)\n\tat org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)\n\tat org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)\n\tat org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)\n\tat org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)\n\tat org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)\n\tat org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)\n\tat org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)\n\tat org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)\n\tat org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:120)\n\tat org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)\n\tat org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)\n\tat org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)\n\tat org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)\n\tat org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)\n\tat org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)\n\tat org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)\n\tat org.apache.solr.servlet.HttpSolrCall.remoteQuery(HttpSolrCall.java:707)\n\t... 30 more\n",
"code":500}}
at __randomizedtesting.SeedInfo.seed([A2E34C95B48671C0:2AB7734F1A7A1C38]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:629)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:265)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:504)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:479)
at org.apache.solr.cloud.hdfs.StressHdfsTest.createAndDeleteCollection(StressHdfsTest.java:189)
at org.apache.solr.cloud.hdfs.StressHdfsTest.test(StressHdfsTest.java:103)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessorTest.testPreemptiveCreation
Error Message:
waiting for collections to be created Timeout waiting to see state for collection=myalias__TRA__2017-10-27 :null Live Nodes: [127.0.0.1:34784_solr, 127.0.0.1:35463_solr, 127.0.0.1:39762_solr, 127.0.0.1:45454_solr] Last available state: null
Stack Trace:
java.lang.AssertionError: waiting for collections to be created
Timeout waiting to see state for collection=myalias__TRA__2017-10-27 :null
Live Nodes: [127.0.0.1:34784_solr, 127.0.0.1:35463_solr, 127.0.0.1:39762_solr, 127.0.0.1:45454_solr]
Last available state: null
at __randomizedtesting.SeedInfo.seed([A2E34C95B48671C0:CF89C93E8CE49CF4]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:321)
at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:298)
at org.apache.solr.update.processor.RoutedAliasUpdateProcessorTest.waitCol(RoutedAliasUpdateProcessorTest.java:241)
at org.apache.solr.update.processor.RoutedAliasUpdateProcessorTest.waitColAndAlias(RoutedAliasUpdateProcessorTest.java:72)
at org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessorTest.testPreemptiveCreation(TimeRoutedAliasUpdateProcessorTest.java:412)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 15366 lines...]
[junit4] Suite: org.apache.solr.cloud.LegacyCloudClusterPropTest
[junit4] 2> 4362895 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LegacyCloudClusterPropTest_A2E34C95B48671C0-001/data-dir-198-001
[junit4] 2> 4362895 WARN (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=21 numCloses=21
[junit4] 2> 4362895 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 4362897 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 4362897 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 4362897 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LegacyCloudClusterPropTest_A2E34C95B48671C0-001/tempDir-001
[junit4] 2> 4362898 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 4362898 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 4362898 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 4362998 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:34037
[junit4] 2> 4362998 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:34037
[junit4] 2> 4362998 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:34037
[junit4] 2> 4362998 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 34037
[junit4] 2> 4363000 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 4363002 INFO (zkConnectionManagerCallback-18742-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4363002 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 4363005 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 4363006 INFO (zkConnectionManagerCallback-18744-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4363006 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 4363007 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 4363007 INFO (zkConnectionManagerCallback-18746-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4363007 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 4363111 WARN (jetty-launcher-18747-thread-1) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 4363111 INFO (jetty-launcher-18747-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 4363111 INFO (jetty-launcher-18747-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 4363111 INFO (jetty-launcher-18747-thread-1) [ ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
[junit4] 2> 4363118 INFO (jetty-launcher-18747-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 4363118 INFO (jetty-launcher-18747-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 4363118 INFO (jetty-launcher-18747-thread-1) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 4363118 INFO (jetty-launcher-18747-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@55761bb6{/solr,null,AVAILABLE}
[junit4] 2> 4363119 INFO (jetty-launcher-18747-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@239aa992{SSL,[ssl, http/1.1]}{127.0.0.1:33724}
[junit4] 2> 4363119 INFO (jetty-launcher-18747-thread-1) [ ] o.e.j.s.Server Started @4363279ms
[junit4] 2> 4363119 INFO (jetty-launcher-18747-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33724}
[junit4] 2> 4363119 ERROR (jetty-launcher-18747-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4363119 INFO (jetty-launcher-18747-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 4363119 INFO (jetty-launcher-18747-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.4.0
[junit4] 2> 4363119 INFO (jetty-launcher-18747-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4363119 INFO (jetty-launcher-18747-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 4363119 INFO (jetty-launcher-18747-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-11-29T19:46:29.591Z
[junit4] 2> 4363121 INFO (jetty-launcher-18747-thread-1) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 4363121 INFO (zkConnectionManagerCallback-18749-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4363121 INFO (jetty-launcher-18747-thread-1) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 4363122 INFO (jetty-launcher-18747-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 4363273 INFO (jetty-launcher-18747-thread-1) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 4363273 WARN (jetty-launcher-18747-thread-1) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 4363274 WARN (jetty-launcher-18747-thread-1) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2e3d0dea[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 4363274 WARN (jetty-launcher-18747-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2e3d0dea[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 4363276 WARN (jetty-launcher-18747-thread-1) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 4363277 WARN (jetty-launcher-18747-thread-1) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@556284b8[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 4363277 WARN (jetty-launcher-18747-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@556284b8[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 4363278 INFO (jetty-launcher-18747-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34037/solr
[junit4] 2> 4363279 INFO (jetty-launcher-18747-thread-1) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 4363280 INFO (zkConnectionManagerCallback-18756-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4363280 INFO (jetty-launcher-18747-thread-1) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 4363382 INFO (jetty-launcher-18747-thread-1) [n:127.0.0.1:33724_solr ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 4363383 INFO (zkConnectionManagerCallback-18758-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4363383 INFO (jetty-launcher-18747-thread-1) [n:127.0.0.1:33724_solr ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 4363455 INFO (jetty-launcher-18747-thread-1) [n:127.0.0.1:33724_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33724_solr
[junit4] 2> 4363455 INFO (jetty-launcher-18747-thread-1) [n:127.0.0.1:33724_solr ] o.a.s.c.Overseer Overseer (id=75818979694018565-127.0.0.1:33724_solr-n_0000000000) starting
[junit4] 2> 4363461 INFO (OverseerStateUpdate-75818979694018565-127.0.0.1:33724_solr-n_0000000000) [n:127.0.0.1:33724_solr ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:33724_solr
[junit4] 2> 4363462 INFO (jetty-launcher-18747-thread-1) [n:127.0.0.1:33724_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33724_solr
[junit4] 2> 4363463 INFO (zkCallback-18757-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4363467 DEBUG (OverseerAutoScalingTriggerThread-75818979694018565-127.0.0.1:33724_solr-n_0000000000) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:33724_solr]
[junit4] 2> 4363480 DEBUG (ScheduledTrigger-14567-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 4363480 INFO (jetty-launcher-18747-thread-1) [n:127.0.0.1:33724_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 4363505 INFO (jetty-launcher-18747-thread-1) [n:127.0.0.1:33724_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33724.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@27cc28e9
[junit4] 2> 4363513 INFO (jetty-launcher-18747-thread-1) [n:127.0.0.1:33724_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33724.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@27cc28e9
[junit4] 2> 4363513 INFO (jetty-launcher-18747-thread-1) [n:127.0.0.1:33724_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33724.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@27cc28e9
[junit4] 2> 4363514 INFO (jetty-launcher-18747-thread-1) [n:127.0.0.1:33724_solr ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 4363515 INFO (jetty-launcher-18747-thread-1) [n:127.0.0.1:33724_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LegacyCloudClusterPropTest_A2E34C95B48671C0-001/tempDir-001/node1/.
[junit4] 2> 4363531 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=1
[junit4] 2> 4363531 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 4363539 INFO (zkConnectionManagerCallback-18767-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4363539 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 4363540 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4363542 INFO (SUITE-LegacyCloudClusterPropTest-seed#[A2E34C95B48671C0]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34037/solr ready
[junit4] 2> 4363564 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testCreateCollectionSwitchLegacyCloud
[junit4] 2> 4363576 INFO (qtp1818247253-68099) [n:127.0.0.1:33724_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=true&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4363583 INFO (qtp1818247253-68099) [n:127.0.0.1:33724_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=true&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=7
[junit4] 2> 4363584 INFO (qtp1818247253-68096) [n:127.0.0.1:33724_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf&maxShardsPerNode=1&name=legacyTrue&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4363587 INFO (OverseerThreadFactory-14569-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection legacyTrue
[junit4] 2> 4363696 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr x:legacyTrue_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=legacyTrue_shard1_replica_n1&action=CREATE&numShards=1&collection=legacyTrue&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 4363696 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr x:legacyTrue_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 4364482 DEBUG (ScheduledTrigger-14567-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 4364707 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.4.0
[junit4] 2> 4364715 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.s.IndexSchema [legacyTrue_shard1_replica_n1] Schema name=minimal
[junit4] 2> 4364718 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 4364718 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'legacyTrue_shard1_replica_n1' using configuration from collection legacyTrue, trusted=true
[junit4] 2> 4364720 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33724.solr.core.legacyTrue.shard1.replica_n1' (registry 'solr.core.legacyTrue.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@27cc28e9
[junit4] 2> 4364720 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.SolrCore [[legacyTrue_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LegacyCloudClusterPropTest_A2E34C95B48671C0-001/tempDir-001/node1/legacyTrue_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LegacyCloudClusterPropTest_A2E34C95B48671C0-001/tempDir-001/node1/./legacyTrue_shard1_replica_n1/data/]
[junit4] 2> 4364764 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 4364764 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 4364765 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 4364765 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 4364766 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@7bf370b9[legacyTrue_shard1_replica_n1] main]
[junit4] 2> 4364768 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 4364768 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 4364768 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 4364768 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1651566749931274240
[junit4] 2> 4364771 INFO (searcherExecutor-14574-thread-1-processing-n:127.0.0.1:33724_solr x:legacyTrue_shard1_replica_n1 c:legacyTrue s:shard1) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.SolrCore [legacyTrue_shard1_replica_n1] Registered new searcher Searcher@7bf370b9[legacyTrue_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 4364774 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/legacyTrue/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 4364774 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/legacyTrue/leaders/shard1
[junit4] 2> 4364777 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 4364777 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 4364777 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:33724/solr/legacyTrue_shard1_replica_n1/
[junit4] 2> 4364778 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 4364778 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.SyncStrategy https://127.0.0.1:33724/solr/legacyTrue_shard1_replica_n1/ has no replicas
[junit4] 2> 4364778 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/legacyTrue/leaders/shard1/leader after winning as /collections/legacyTrue/leader_elect/shard1/election/75818979694018565-core_node2-n_0000000000
[junit4] 2> 4364780 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:33724/solr/legacyTrue_shard1_replica_n1/ shard1
[junit4] 2> 4364881 INFO (zkCallback-18757-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/legacyTrue/state.json] for collection [legacyTrue] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4364882 INFO (zkCallback-18757-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/legacyTrue/state.json] for collection [legacyTrue] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4364882 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 4364884 INFO (qtp1818247253-68098) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=legacyTrue_shard1_replica_n1&action=CREATE&numShards=1&collection=legacyTrue&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1188
[junit4] 2> 4364886 INFO (qtp1818247253-68096) [n:127.0.0.1:33724_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 4364984 INFO (zkCallback-18757-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/legacyTrue/state.json] for collection [legacyTrue] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4364984 INFO (zkCallback-18757-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/legacyTrue/state.json] for collection [legacyTrue] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4364984 INFO (zkCallback-18757-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/legacyTrue/state.json] for collection [legacyTrue] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4364985 INFO (qtp1818247253-68096) [n:127.0.0.1:33724_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf&maxShardsPerNode=1&name=legacyTrue&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=1400
[junit4] 2> 4364985 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.MiniSolrCloudCluster waitForActiveCollection: legacyTrue
[junit4] 2> 4364991 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=705780899
[junit4] 2> 4364991 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:33724_solr
[junit4] 2> 4364991 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.ZkController Publish this node as DOWN...
[junit4] 2> 4364991 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.ZkController Publish node=127.0.0.1:33724_solr as DOWN
[junit4] 2> 4364993 INFO (coreCloseExecutor-14579-thread-1) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 r:core_node2 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.SolrCore [legacyTrue_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@3d8b30f2
[junit4] 2> 4364993 INFO (coreCloseExecutor-14579-thread-1) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 r:core_node2 x:legacyTrue_shard1_replica_n1 ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.legacyTrue.shard1.replica_n1, tag=SolrCore@3d8b30f2
[junit4] 2> 4364993 INFO (coreCloseExecutor-14579-thread-1) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 r:core_node2 x:legacyTrue_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@29e83e34: rootName = solr_33724, domain = solr.core.legacyTrue.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.legacyTrue.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@61e5d170
[junit4] 2> 4365000 INFO (coreCloseExecutor-14579-thread-1) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 r:core_node2 x:legacyTrue_shard1_replica_n1 ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.legacyTrue.shard1.leader, tag=SolrCore@3d8b30f2
[junit4] 2> 4365002 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 4365003 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5ea55c95: rootName = solr_33724, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@5088eb73
[junit4] 2> 4365008 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 4365008 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@51b78a5c: rootName = solr_33724, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@413a9864
[junit4] 2> 4365011 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 4365011 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2639ff88: rootName = solr_33724, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@7891c44
[junit4] 2> 4365011 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 4365018 INFO (closeThreadPool-18772-thread-1) [ ] o.a.s.c.Overseer Overseer (id=75818979694018565-127.0.0.1:33724_solr-n_0000000000) closing
[junit4] 2> 4365018 INFO (OverseerStateUpdate-75818979694018565-127.0.0.1:33724_solr-n_0000000000) [n:127.0.0.1:33724_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:33724_solr
[junit4] 2> 4365018 INFO (OverseerAutoScalingTriggerThread-75818979694018565-127.0.0.1:33724_solr-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 4365032 INFO (closeThreadPool-18772-thread-2) [ ] o.a.s.c.Overseer Overseer (id=75818979694018565-127.0.0.1:33724_solr-n_0000000000) closing
[junit4] 2> 4365033 INFO (zkCallback-18766-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 4365133 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.Overseer Overseer (id=75818979694018565-127.0.0.1:33724_solr-n_0000000000) closing
[junit4] 2> 4365146 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@239aa992{SSL,[ssl, http/1.1]}{127.0.0.1:0}
[junit4] 2> 4365146 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@55761bb6{/solr,null,UNAVAILABLE}
[junit4] 2> 4365147 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.e.j.s.session node0 Stopped scavenging
[junit4] 2> 4365147 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.MiniSolrCloudCluster waitForJettyToStop: 33724
[junit4] 2> 4365147 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=33724)
[junit4] 2> 4365148 WARN (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 4365148 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 33724 try number 1 ...
[junit4] 2> 4365148 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.e.j.s.Server jetty-9.4.19.v20190610; built: 2019-06-10T16:30:51.723Z; git: afcf563148970e98786327af5e07c261fda175d3; jvm 1.8.0_191-b12
[junit4] 2> 4365149 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 4365149 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 4365149 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 4365149 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2b09ed05{/solr,null,AVAILABLE}
[junit4] 2> 4365150 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7b64858b{SSL,[ssl, http/1.1]}{127.0.0.1:33724}
[junit4] 2> 4365150 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.e.j.s.Server Started @4365310ms
[junit4] 2> 4365150 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33724}
[junit4] 2> 4365150 ERROR (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4365150 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 4365150 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.4.0
[junit4] 2> 4365150 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4365150 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 4365150 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-11-29T19:46:31.622Z
[junit4] 2> 4365151 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 4365152 INFO (zkConnectionManagerCallback-18774-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4365152 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 4365153 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 4365341 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 4365342 WARN (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 4365343 WARN (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@7078123e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 4365343 WARN (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@7078123e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 4365346 WARN (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 4365346 WARN (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@260ea11b[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 4365346 WARN (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@260ea11b[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 4365347 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34037/solr
[junit4] 2> 4365348 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 4365349 INFO (zkConnectionManagerCallback-18781-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4365349 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 4365451 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [n:127.0.0.1:33724_solr ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 4365452 INFO (zkConnectionManagerCallback-18783-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4365452 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [n:127.0.0.1:33724_solr ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 4365459 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [n:127.0.0.1:33724_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33724_solr
[junit4] 2> 4365460 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [n:127.0.0.1:33724_solr ] o.a.s.c.Overseer Overseer (id=75818979694018569-127.0.0.1:33724_solr-n_0000000001) starting
[junit4] 2> 4365464 INFO (OverseerStateUpdate-75818979694018569-127.0.0.1:33724_solr-n_0000000001) [n:127.0.0.1:33724_solr ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:33724_solr
[junit4] 2> 4365464 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [n:127.0.0.1:33724_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33724_solr
[junit4] 2> 4365465 INFO (zkCallback-18766-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4365466 INFO (zkCallback-18782-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4365466 DEBUG (OverseerAutoScalingTriggerThread-75818979694018569-127.0.0.1:33724_solr-n_0000000001) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:33724_solr]
[junit4] 2> 4365476 DEBUG (ScheduledTrigger-14587-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 4365483 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [n:127.0.0.1:33724_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 4365501 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [n:127.0.0.1:33724_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33724.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@27cc28e9
[junit4] 2> 4365510 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [n:127.0.0.1:33724_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33724.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@27cc28e9
[junit4] 2> 4365510 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [n:127.0.0.1:33724_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33724.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@27cc28e9
[junit4] 2> 4365512 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [n:127.0.0.1:33724_solr ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 4365513 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [n:127.0.0.1:33724_solr ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LegacyCloudClusterPropTest_A2E34C95B48671C0-001/tempDir-001/node1/.
[junit4] 2> 4365513 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [n:127.0.0.1:33724_solr ] o.a.s.c.CorePropertiesLocator Cores are: [legacyTrue_shard1_replica_n1]
[junit4] 2> 4365514 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 4365521 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.4.0
[junit4] 2> 4365529 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr ] o.a.s.s.IndexSchema [legacyTrue_shard1_replica_n1] Schema name=minimal
[junit4] 2> 4365531 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 4365538 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.4.0
[junit4] 2> 4365546 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.s.IndexSchema [legacyTrue_shard1_replica_n1] Schema name=minimal
[junit4] 2> 4365548 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 4365548 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'legacyTrue_shard1_replica_n1' using configuration from collection legacyTrue, trusted=true
[junit4] 2> 4365549 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33724.solr.core.legacyTrue.shard1.replica_n1' (registry 'solr.core.legacyTrue.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@27cc28e9
[junit4] 2> 4365549 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.SolrCore [[legacyTrue_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LegacyCloudClusterPropTest_A2E34C95B48671C0-001/tempDir-001/node1/legacyTrue_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.LegacyCloudClusterPropTest_A2E34C95B48671C0-001/tempDir-001/node1/./legacyTrue_shard1_replica_n1/data/]
[junit4] 2> 4365585 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 4365586 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 4365586 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 4365586 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 4365587 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@219510ee[legacyTrue_shard1_replica_n1] main]
[junit4] 2> 4365588 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 4365589 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 4365589 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 4365589 INFO (coreLoadExecutor-14593-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1651566750792155136
[junit4] 2> 4365591 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=1
[junit4] 2> 4365592 INFO (searcherExecutor-14594-thread-1-processing-n:127.0.0.1:33724_solr x:legacyTrue_shard1_replica_n1 c:legacyTrue s:shard1) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.SolrCore [legacyTrue_shard1_replica_n1] Registered new searcher Searcher@219510ee[legacyTrue_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 4365601 INFO (coreZkRegister-14581-thread-1-processing-n:127.0.0.1:33724_solr x:legacyTrue_shard1_replica_n1 c:legacyTrue s:shard1) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 r:core_node2 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/legacyTrue/leaders/shard1
[junit4] 2> 4365602 INFO (coreZkRegister-14581-thread-1-processing-n:127.0.0.1:33724_solr x:legacyTrue_shard1_replica_n1 c:legacyTrue s:shard1) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 r:core_node2 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 4365602 INFO (coreZkRegister-14581-thread-1-processing-n:127.0.0.1:33724_solr x:legacyTrue_shard1_replica_n1 c:legacyTrue s:shard1) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 r:core_node2 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 4365602 INFO (coreZkRegister-14581-thread-1-processing-n:127.0.0.1:33724_solr x:legacyTrue_shard1_replica_n1 c:legacyTrue s:shard1) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 r:core_node2 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:33724/solr/legacyTrue_shard1_replica_n1/
[junit4] 2> 4365602 INFO (coreZkRegister-14581-thread-1-processing-n:127.0.0.1:33724_solr x:legacyTrue_shard1_replica_n1 c:legacyTrue s:shard1) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 r:core_node2 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 4365603 INFO (coreZkRegister-14581-thread-1-processing-n:127.0.0.1:33724_solr x:legacyTrue_shard1_replica_n1 c:legacyTrue s:shard1) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 r:core_node2 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.SyncStrategy https://127.0.0.1:33724/solr/legacyTrue_shard1_replica_n1/ has no replicas
[junit4] 2> 4365603 INFO (coreZkRegister-14581-thread-1-processing-n:127.0.0.1:33724_solr x:legacyTrue_shard1_replica_n1 c:legacyTrue s:shard1) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 r:core_node2 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/legacyTrue/leaders/shard1/leader after winning as /collections/legacyTrue/leader_elect/shard1/election/75818979694018569-core_node2-n_0000000001
[junit4] 2> 4365604 INFO (coreZkRegister-14581-thread-1-processing-n:127.0.0.1:33724_solr x:legacyTrue_shard1_replica_n1 c:legacyTrue s:shard1) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 r:core_node2 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:33724/solr/legacyTrue_shard1_replica_n1/ shard1
[junit4] 2> 4365605 INFO (coreZkRegister-14581-thread-1-processing-n:127.0.0.1:33724_solr x:legacyTrue_shard1_replica_n1 c:legacyTrue s:shard1) [n:127.0.0.1:33724_solr c:legacyTrue s:shard1 r:core_node2 x:legacyTrue_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 4365707 INFO (zkCallback-18782-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/legacyTrue/state.json] for collection [legacyTrue] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4365707 INFO (zkCallback-18782-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/legacyTrue/state.json] for collection [legacyTrue] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4365707 INFO (zkCallback-18782-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/legacyTrue/state.json] for collection [legacyTrue] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4366478 DEBUG (ScheduledTrigger-14587-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 4366601 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.h.i.e.RetryExec I/O exception (org.apache.http.NoHttpResponseException) caught when processing request to {s}->https://127.0.0.1:33724: The target server failed to respond
[junit4] 2> 4366601 INFO (TEST-LegacyCloudClusterPropTest.testCreateCollectionSwitchLegacyCloud-seed#[A2E34C95B48671C0]) [ ] o.a.h.i.e.RetryExec Retrying request to {s}->https://127.0.0.1:33724
[junit4] 2> 4366616 INFO (qtp974242673-68163) [n:127.0.0.1:33724_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4366617 INFO (qtp974242673-68163) [n:127.0.0.1:33724_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 4366618 INFO (qtp974242673-68159) [n:127.0.0.1:33724_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf&maxShardsPerNode=1&name=legacyFalse&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4366621 INFO (OverseerThreadFactory-14589-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection legacyFalse
[junit4] 2> 4366830 INFO (qtp974242673-68162) [n:127.0.0.1:33724_solr x:legacyFalse_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=legacyFalse_shard1_replica_n1&action=CREATE&numShards=1&collection=legacyFalse&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 4366834 ERROR (qtp974242673-68162) [n:127.0.0.1:33724_solr x:legacyFalse_shard1_replica_n1 ] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'legacyFalse_shard1_replica_n1': non legacy mode coreNodeName missing {collection.configName=conf, numShards=1, shard=shard1, collection=legacyFalse, replicaType=NRT}
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1225)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:93)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:362)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:198)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:839)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:805)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:558)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:419)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1711)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1347)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1678)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1249)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
[junit4] 2> at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:505)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
[junit4] 2> at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:427)
[junit4] 2> at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:321)
[junit4] 2> at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
[junit4] 2> at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.solr.common.SolrException: non legacy mode coreNodeName missing {collection.configName=conf, numShards=1, shard=shard1, collection=legacyFalse, replicaType=NRT}
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1172)
[junit4] 2> ... 41 more
[junit4] 2>
[junit4] 2> 4366834 INFO (qtp974242673-68162) [n:127.0.0.1:33724_solr x:legacyFalse_shard1_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=legacyFalse_shard1_replica_n1&action=CREATE&numShards=1&collection=legacyFalse&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=400 QTime=3
[junit4] 2> 4366834 ERROR (OverseerThreadFactory-14589-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr ] o.a.s.c.a.c.OverseerCollectionMessageHandler Error from shard: https://127.0.0.1:33724/solr
[junit4] 2> => org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:33724/solr: Error CREATEing SolrCore 'legacyFalse_shard1_replica_n1': non legacy mode coreNodeName missing {collection.configName=conf, numShards=1, shard=shard1, collection=legacyFalse, replicaType=NRT}
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:665)
[junit4] 2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:33724/solr: Error CREATEing SolrCore 'legacyFalse_shard1_replica_n1': non legacy mode coreNodeName missing {collection.configName=conf, numShards=1, shard=shard1, collection=legacyFalse, replicaType=NRT}
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:665) ~[java/:?]
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:265) ~[java/:?]
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248) ~[java/:?]
[junit4] 2> at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1290) ~[java/:?]
[junit4] 2> at org.apache.solr.handler.component.HttpShardHandlerFactory$1.request(HttpShardHandlerFactory.java:176) ~[java/:?]
[junit4] 2> at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:200) ~[java/:?]
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_191]
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_191]
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_191]
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:181) ~[metrics-core-4.0.5.jar:4.0.5]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) ~[java/:?]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_191]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_191]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
[junit4] 2> 4366834 ERROR (OverseerThreadFactory-14589-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr ] o.a.s.c.a.c.OverseerCollectionMessageHandler Cleaning up collection [legacyFalse].
[junit4] 2> 4366835 INFO (OverseerThreadFactory-14589-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr ] o.a.s.c.a.c.OverseerCollectionMessageHandler Executing Collection Cmd=action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true&deleteMetricsHistory=true, asyncId=null
[junit4] 2> 4366938 INFO (zkCallback-18782-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDeleted path:/collections/legacyFalse/state.json] for collection [legacyFalse] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4366941 INFO (OverseerThreadFactory-14589-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr ] o.a.s.c.a.c.CreateCollectionCmd Cleaned up artifacts for failed create collection for [legacyFalse]
[junit4] 2> 4366941 ERROR (OverseerThreadFactory-14589-thread-1-processing-n:127.0.0.1:33724_solr) [n:127.0.0.1:33724_solr ] o.a.s.c.a.c.OverseerCollectionMessageHandler Collection: legacyFalse operation: create failed:org.apache.solr.common.SolrException: Underlying core creation failed while creating collection: legacyFalse
[junit4] 2> at org.apache.solr.cloud.api.collections.CreateCollectionCmd.call(CreateCollectionCmd.java:303)
[junit4] 2> at org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:263)
[junit4] 2> at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:505)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2>
[junit4] 2> 436695
[...truncated too long message...]
ect/election/75819209728851976-127.0.0.1:35612_solr-n_0000000000
[junit4] 2> 2 /solr/collections/myalias__TRA__2017-10-25/leader_elect/shard3/election/75819209728851980-core_node18-n_0000000000
[junit4] 2> 2 /solr/collections/myalias__TRA__2017-10-24/leader_elect/shard2/election/75819209728851980-core_node13-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 126 /solr/overseer/queue
[junit4] 2> 30 /solr/collections
[junit4] 2> 22 /solr/live_nodes
[junit4] 2> 17 /solr/overseer/collection-queue-work
[junit4] 2>
[junit4] 2> 7899136 INFO (TEST-TimeRoutedAliasUpdateProcessorTest.testSliceRouting-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:45130
[junit4] 2> 7899136 INFO (TEST-TimeRoutedAliasUpdateProcessorTest.testSliceRouting-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:45130
[junit4] 2> 7899136 INFO (TEST-TimeRoutedAliasUpdateProcessorTest.testSliceRouting-seed#[A2E34C95B48671C0]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 45130
[junit4] 2> 7899138 INFO (TEST-TimeRoutedAliasUpdateProcessorTest.testSliceRouting-seed#[A2E34C95B48671C0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSliceRouting
[junit4] IGNOR/A 0.00s J2 | TimeRoutedAliasUpdateProcessorTest.testDateMathInStart
[junit4] > Assumption #1: 'awaitsfix' test group is disabled (@AwaitsFix(bugUrl=https://issues.apache.org/jira/browse/SOLR-13943))
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.update.processor.TimeRoutedAliasUpdateProcessorTest_A2E34C95B48671C0-001
[junit4] 2> Nov 29, 2019 8:45:25 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene84): {}, docValues:{}, maxPointsInLeafNode=1678, maxMBSortInHeap=6.221097374754438, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@5b7d5e50), locale=it-IT, timezone=IET
[junit4] 2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=139181704,total=510656512
[junit4] 2> NOTE: All tests run in this JVM: [TestSolrCLIRunExample, SolrTestCaseJ4Test, HdfsChaosMonkeyNothingIsSafeTest, TestCloudNestedDocsSort, FileUtilsTest, DistributedQueryElevationComponentTest, JSONWriterTest, DocumentBuilderTest, TestDocSet, PKIAuthenticationIntegrationTest, TestLeaderElectionWithEmptyReplica, TestTlogReplica, RollingRestartTest, TestFacetMethods, TestJsonFacetsWithNestedObjects, TestSolrQueryParser, TestDistribDocBasedVersion, ConfigSetsAPITest, TestCloudPhrasesIdentificationComponent, TestCloudConsistency, SortSpecParsingTest, TestMultiWordSynonyms, CurrencyFieldTypeTest, XMLAtomicUpdateMultivalueTest, TestLFUCache, TestFieldCollectionResource, TestDistributedGrouping, DistributedMLTComponentTest, TestFastLRUCache, CheckHdfsIndexTest, TestMiniSolrCloudClusterSSL, TestLegacyFieldCache, DistributedFacetExistsSmallTest, TestCloudSchemaless, SpatialHeatmapFacetsTest, TestTrackingShardHandlerFactory, CoreAdminOperationTest, TestSimLargeCluster, UnloadDistributedZkTest, BinaryUpdateRequestHandlerTest, DocValuesNotIndexedTest, TestSimPolicyCloud, TestSnapshotCloudManager, IndexSchemaTest, TestChildDocTransformerHierarchy, TestConfigSetsAPIZkFailure, TestClusterStateMutator, TestReplicationHandlerBackup, TestCustomStream, FullSolrCloudDistribCmdsTest, TestCaffeineCache, ZkStateWriterTest, TestExactStatsCache, UUIDFieldTest, TestTolerantSearch, TestPayloadCheckQParserPlugin, SolrCoreMetricManagerTest, SimpleCollectionCreateDeleteTest, TestBlobHandler, TestPullReplica, TokenizerChainTest, SyncSliceTest, TestClassicSimilarityFactory, BasicAuthStandaloneTest, ReindexCollectionTest, TestRestoreCore, SolrRequestParserTest, BlockDirectoryTest, TestSearcherReuse, DeleteStatusTest, TestStressUserVersions, HighlighterTest, CreateRoutedAliasTest, BasicDistributedZkTest, TestQuerySenderListener, TestPKIAuthenticationPlugin, TestBinaryField, ClusterStateTest, DistribJoinFromCollectionTest, DeleteNodeTest, CollectionsAPIAsyncDistributedZkTest, TestInfoStreamLogging, SolrCoreTest, TestMinMaxOnMultiValuedField, TestHighFrequencyDictionaryFactory, TestAnalyzeInfixSuggestions, TestCloudRecovery, IgnoreLargeDocumentProcessorFactoryTest, SecurityConfHandlerTest, TestObjectReleaseTracker, LeaderElectionIntegrationTest, RequiredFieldsTest, TaggerTest, MigrateRouteKeyTest, MergeStrategyTest, BadComponentTest, UtilsToolTest, TestScoreJoinQPScore, TestRequestForwarding, RulesTest, EnumFieldTest, PreAnalyzedFieldManagedSchemaCloudTest, BooleanFieldTest, ZkShardTermsTest, DocExpirationUpdateProcessorFactoryTest, TestRandomFlRTGCloud, HttpSolrCallGetCoreTest, TestStressLucene, DOMUtilTest, ShardsWhitelistTest, HdfsSyncSliceTest, ConvertedLegacyTest, TestCoreBackup, TestSolrXml, PeerSyncWithBufferUpdatesTest, DebugComponentTest, EmbeddedSolrNoSerializeTest, BigEndianAscendingWordDeserializerTest, TestSuggestSpellingConverter, FileBasedSpellCheckerTest, DaemonStreamApiTest, ConnectionReuseTest, TestZkAclsWithHadoopAuth, SaslZkACLProviderTest, ProtectedTermFilterFactoryTest, TestInPlaceUpdatesRequiredField, TestLazyCores, SpellCheckCollatorWithCollapseTest, CdcrBootstrapTest, MultiThreadedOCPTest, CloudMLTQParserTest, PeerSyncReplicationTest, TestZkChroot, TestRemoteStreaming, CollectionStateFormat2Test, TestFiltering, TestFieldTypeResource, TestSchemaSimilarityResource, CurrencyRangeFacetCloudTest, TestDocumentBuilder, TestDynamicLoadingUrl, TestFieldCacheSortRandom, AutoscalingHistoryHandlerTest, MultiSolrCloudTestCaseTest, OverseerStatusTest, OverseerTaskQueueTest, PackageManagerCLITest, RecoveryAfterSoftCommitTest, ReplaceNodeTest, SliceStateTest, SolrCLIZkUtilsTest, SolrCloudExampleTest, SolrXmlInZkTest, SplitShardTest, TestCloudDeleteByQuery, TestCloudInspectUtil, TestCloudRecovery2, TestCloudSearcherWarming, TestDistributedMap, TestDownShardTolerantSearch, TestExclusionRuleCollectionAccess, TestHashPartitioner, TestPrepRecovery, TestTolerantUpdateProcessorCloud, TestTolerantUpdateProcessorRandomCloud, TestUtilizeNode, VMParamsZkACLAndCredentialsProvidersTest, ZkCLITest, ZkControllerTest, ZkFailoverTest, AssignTest, AsyncCallRequestStatusResponseTest, CollectionReloadTest, CollectionTooManyReplicasTest, ConcurrentDeleteAndCreateCollectionTest, SplitByPrefixTest, TestCollectionsAPIViaSolrCloudCluster, TestReplicaProperties, AutoAddReplicasIntegrationTest, IndexSizeTriggerTest, TriggerIntegrationTest, CdcrReplicationHandlerTest, CdcrRequestHandlerTest, HdfsRecoverLeaseTest, HdfsThreadLeakTest, HdfsTlogReplayBufferedWhileIndexingTest, HdfsWriteToMultipleCollectionsTest, ZkCollectionPropsCachingTest, ZkStateReaderTest, ImplicitSnitchTest, RuleEngineTest, AlternateDirectoryTest, BlobRepositoryCloudTest, ByteBuffersDirectoryFactoryTest, CachingDirectoryFactoryTest, ConfigureRecoveryStrategyTest, ExitableDirectoryReaderTest, HdfsDirectoryFactoryTest, QueryResultKeyTest, TestDirectoryFactory, TestDynamicLoading, TestSolrCoreSnapshots, TestDistribPackageStore, TestSolrConfigHandlerCloud, CoreAdminRequestStatusTest, MetricsHandlerTest, StatsReloadRaceTest, DistributedFacetPivotSmallAdvancedTest, DistributedFacetPivotWhiteBoxTest, FacetPivotSmallTest, InfixSuggestersTest, MoreLikeThisComponentTest, ResponseBuilderTest, SpellCheckComponentTest, SuggestComponentContextFilterQueryTest, SuggestComponentTest, TermVectorComponentDistributedTest, HighlighterConfigTest, HighlighterMaxOffsetTest, TestUnifiedSolrHighlighter, WrapperMergePolicyFactoryTest, TestLegacyTerms, TestMultiValuedNumericRangeQuery, TestNumericRangeQuery64, SolrCloudReportersTest, RegexBytesRefFilterTest, SimpleFacetsTest, SubstringBytesRefFilterTest, TestFaceting, TestIntervalFaceting, TestManagedSchemaAPI, TestManagedSchemaThreadSafety, TestPointFields, TestOverriddenPrefixQueryForCustomFieldType, TestQueryUtils, TestRTGBase, TestRangeQuery, TestReRankQParserPlugin, TestRealTimeGet, TestRecovery, TestReload, TestReloadDeadlock, TestSolrCoreParser, TestSort, TestStandardQParsers, TestTrieFacet, TestValueSourceCache, TestXmlQParser, TestXmlQParserPlugin, DistributedFacetSimpleRefinementLongTailTest, TestJsonFacetRefinement, TestOrdValues, TestSortByMinMaxFunction, BlockJoinFacetRandomTest, BlockJoinFacetSimpleTest, SimpleMLTQParserTest, TestDFISimilarityFactory, TestIBSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestPerFieldSimilarityWithDefaultOverride, AuditLoggerPluginTest, JWTAuthPluginIntegrationTest, TestImpersonationWithHadoopAuth, CacheHeaderTest, DirectSolrConnectionTest, ConjunctionSolrSpellCheckerTest, DirectSolrSpellCheckerTest, SpellingQueryConverterTest, SuggesterFSTTest, SuggesterTSTTest, TestAnalyzedSuggestions, BufferStoreTest, HdfsLockFactoryTest, TestFieldCache, TestFieldCacheReopen, TestFieldCacheSort, TestFieldCacheVsDocValues, TestNumericTerms64, SolrIndexSplitterTest, TestExceedMaxTermLength, TestHdfsUpdateLog, TestIndexingPerformance, TestNestedUpdateProcessor, TestUpdate, TransactionLogTest, UpdateLogTest, UpdateParamsTest, VersionInfoTest, AddSchemaFieldsUpdateProcessorFactoryTest, AtomicUpdatesTest, ClassificationUpdateProcessorIntegrationTest, ClassificationUpdateProcessorTest, CloneFieldUpdateProcessorFactoryTest, DefaultValueUpdateProcessorTest, ParsingFieldUpdateProcessorsTest, ScriptEngineTest, SignatureUpdateProcessorFactoryTest, StatelessScriptUpdateProcessorFactoryTest, TemplateUpdateProcessorTest, TestDocBasedVersionConstraints, TestNamedUpdateProcessors, TestPartialUpdateDeduplication, TimeRoutedAliasUpdateProcessorTest]
[junit4] Completed [846/887 (3!)] on J2 in 228.69s, 6 tests, 1 failure, 1 skipped <<< FAILURES!
[...truncated 53029 lines...]
[JENKINS] Lucene-Solr-BadApples-NightlyTests-8.x - Build # 49 -
Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-8.x/49/
1 tests failed.
FAILED: org.apache.lucene.document.TestXYMultiPolygonShapeQueries.testRandomBig
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
at __randomizedtesting.SeedInfo.seed([4B8813CF298D533E:CCDF6E40B8D42FBE]:0)
at org.apache.lucene.util.bkd.HeapPointWriter.<init>(HeapPointWriter.java:42)
at org.apache.lucene.util.bkd.BKDRadixSelector.getPointWriter(BKDRadixSelector.java:513)
at org.apache.lucene.util.bkd.BKDRadixSelector.select(BKDRadixSelector.java:123)
at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1647)
at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1669)
at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1669)
at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1664)
at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1664)
at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1664)
at org.apache.lucene.util.bkd.BKDWriter.finish(BKDWriter.java:797)
at org.apache.lucene.index.RandomCodec$1$1.writeField(RandomCodec.java:140)
at org.apache.lucene.codecs.PointsWriter.mergeOneField(PointsWriter.java:62)
at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:191)
at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:143)
at org.apache.lucene.codecs.asserting.AssertingPointsFormat$AssertingPointsWriter.merge(AssertingPointsFormat.java:149)
at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:202)
at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:162)
at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2158)
at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5120)
at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1597)
at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
at org.apache.lucene.document.BaseShapeTestCase.indexRandomShapes(BaseShapeTestCase.java:231)
at org.apache.lucene.document.BaseShapeTestCase.verify(BaseShapeTestCase.java:212)
at org.apache.lucene.document.BaseShapeTestCase.doTestRandom(BaseShapeTestCase.java:138)
at org.apache.lucene.document.TestXYMultiPolygonShapeQueries.testRandomBig(TestXYMultiPolygonShapeQueries.java:143)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
Build Log:
[...truncated 11356 lines...]
[junit4] Suite: org.apache.lucene.document.TestXYMultiPolygonShapeQueries
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestXYMultiPolygonShapeQueries -Dtests.method=testRandomBig -Dtests.seed=4B8813CF298D533E -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.badapples=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=ro-RO -Dtests.timezone=Asia/Famagusta -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 973s J1 | TestXYMultiPolygonShapeQueries.testRandomBig <<<
[junit4] > Throwable #1: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([4B8813CF298D533E:CCDF6E40B8D42FBE]:0)
[junit4] > at org.apache.lucene.util.bkd.HeapPointWriter.<init>(HeapPointWriter.java:42)
[junit4] > at org.apache.lucene.util.bkd.BKDRadixSelector.getPointWriter(BKDRadixSelector.java:513)
[junit4] > at org.apache.lucene.util.bkd.BKDRadixSelector.select(BKDRadixSelector.java:123)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1647)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1669)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1669)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1664)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1664)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1664)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.finish(BKDWriter.java:797)
[junit4] > at org.apache.lucene.index.RandomCodec$1$1.writeField(RandomCodec.java:140)
[junit4] > at org.apache.lucene.codecs.PointsWriter.mergeOneField(PointsWriter.java:62)
[junit4] > at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:191)
[junit4] > at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:143)
[junit4] > at org.apache.lucene.codecs.asserting.AssertingPointsFormat$AssertingPointsWriter.merge(AssertingPointsFormat.java:149)
[junit4] > at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:202)
[junit4] > at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:162)
[junit4] > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
[junit4] > at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
[junit4] > at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
[junit4] > at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2158)
[junit4] > at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5120)
[junit4] > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1597)
[junit4] > at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
[junit4] > at org.apache.lucene.document.BaseShapeTestCase.indexRandomShapes(BaseShapeTestCase.java:231)
[junit4] > at org.apache.lucene.document.BaseShapeTestCase.verify(BaseShapeTestCase.java:212)
[junit4] > at org.apache.lucene.document.BaseShapeTestCase.doTestRandom(BaseShapeTestCase.java:138)
[junit4] > at org.apache.lucene.document.TestXYMultiPolygonShapeQueries.testRandomBig(TestXYMultiPolygonShapeQueries.java:143)
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/sandbox/test/J1/temp/lucene.document.TestXYMultiPolygonShapeQueries_4B8813CF298D533E-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene84): {id=BlockTreeOrds(blocksize=128)}, docValues:{shape=DocValuesFormat(name=Direct), id=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=928, maxMBSortInHeap=6.872291520847615, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@309ed5de), locale=ro-RO, timezone=Asia/Famagusta
[junit4] 2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=151691928,total=288882688
[junit4] 2> NOTE: All tests run in this JVM: [TestBigIntegerPoint, TestXYMultiPolygonShapeQueries]
[junit4] Completed [35/40 (1!)] on J1 in 1754.89s, 5 tests, 1 error <<< FAILURES!
[...truncated 18 lines...]
[junit4] JVM J1: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/sandbox/test/temp/junit4-J1-20191227_194341_1611762379036369179465.sysout
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/heapdumps/java_pid20764.hprof ...
[junit4] Heap dump file created [451379878 bytes in 0.773 secs]
[junit4] <<< JVM J1: EOF ----
[...truncated 15185 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/build.xml:662: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/build.xml:586: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid20764.hprof
Total time: 506 minutes 55 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-BadApples-NightlyTests-8.x - Build # 48 -
Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-8.x/48/
1 tests failed.
FAILED: org.apache.lucene.search.TestDoubleRangeFieldQueries.testRandomBig
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
at __randomizedtesting.SeedInfo.seed([4E8B4ACB3E2B5857:C9DC3744AF7224D7]:0)
at org.apache.lucene.util.ArrayUtil.growExact(ArrayUtil.java:302)
at org.apache.lucene.codecs.compressing.CompressingStoredFieldsIndexReader.<init>(CompressingStoredFieldsIndexReader.java:75)
at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.<init>(CompressingTermVectorsReader.java:128)
at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsReader(CompressingTermVectorsFormat.java:91)
at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:130)
at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:679)
at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:302)
at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:862)
at org.apache.lucene.util.IOUtils.close(IOUtils.java:89)
at org.apache.lucene.util.IOUtils.close(IOUtils.java:77)
at org.apache.lucene.search.BaseRangeFieldQueryTestCase.verify(BaseRangeFieldQueryTestCase.java:322)
at org.apache.lucene.search.BaseRangeFieldQueryTestCase.doTestRandom(BaseRangeFieldQueryTestCase.java:189)
at org.apache.lucene.search.BaseRangeFieldQueryTestCase.testRandomBig(BaseRangeFieldQueryTestCase.java:77)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
Build Log:
[...truncated 1836 lines...]
[junit4] Suite: org.apache.lucene.search.TestDoubleRangeFieldQueries
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestDoubleRangeFieldQueries -Dtests.method=testRandomBig -Dtests.seed=4E8B4ACB3E2B5857 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.badapples=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=ga-IE -Dtests.timezone=Pacific/Enderbury -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 130s J0 | TestDoubleRangeFieldQueries.testRandomBig <<<
[junit4] > Throwable #1: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([4E8B4ACB3E2B5857:C9DC3744AF7224D7]:0)
[junit4] > at org.apache.lucene.util.ArrayUtil.growExact(ArrayUtil.java:302)
[junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsIndexReader.<init>(CompressingStoredFieldsIndexReader.java:75)
[junit4] > at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.<init>(CompressingTermVectorsReader.java:128)
[junit4] > at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsReader(CompressingTermVectorsFormat.java:91)
[junit4] > at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:130)
[junit4] > at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
[junit4] > at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:679)
[junit4] > at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:302)
[junit4] > at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:862)
[junit4] > at org.apache.lucene.util.IOUtils.close(IOUtils.java:89)
[junit4] > at org.apache.lucene.util.IOUtils.close(IOUtils.java:77)
[junit4] > at org.apache.lucene.search.BaseRangeFieldQueryTestCase.verify(BaseRangeFieldQueryTestCase.java:322)
[junit4] > at org.apache.lucene.search.BaseRangeFieldQueryTestCase.doTestRandom(BaseRangeFieldQueryTestCase.java:189)
[junit4] > at org.apache.lucene.search.BaseRangeFieldQueryTestCase.testRandomBig(BaseRangeFieldQueryTestCase.java:77)
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/core/test/J0/temp/lucene.search.TestDoubleRangeFieldQueries_4E8B4ACB3E2B5857-001
[junit4] 2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=4, maxDocsPerChunk=608, blockSize=1), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=4, blockSize=1)), sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@8646d33), locale=ga-IE, timezone=Pacific/Enderbury
[junit4] 2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=243183856,total=327155712
[junit4] 2> NOTE: All tests run in this JVM: [TestLazyProxSkipping, TestTieredMergePolicy, TestLucene50StoredFieldsFormatHighCompression, TestIntsRef, TestTermVectorsWriter, TestCharTermAttributeImpl, TestMultiPhraseQuery, TestNeedsScores, TestSimpleFSDirectory, TestEarlyTermination, TestSmallFloat, TestLucene80NormsFormatMergeInstance, TestInfoStream, TestLongValuesSource, TestRegExp, TestFieldInvertState, TestLucene60FieldInfoFormat, TestAtomicUpdate, TestManyFields, TestIndexWriterReader, TestFutureObjects, TestMergeSchedulerExternal, TestPriorityQueue, TestLucene84PostingsFormat, TestAxiomaticF2LOG, TestSearcherManager, TestBlockMaxConjunction, TestNRTThreads, TestSimilarity, TestSpanBoostQuery, TestNoDeletionPolicy, TestDocIdsWriter, TestMergeRateLimiter, TestIndexWriterMerging, TestSegmentInfos, TestPolygon, TestIndexWriterLockRelease, TestSimilarity2, TestSimpleExplanationsOfNonMatches, TestCharArraySet, Test4GBStoredFields, TestIntRange, TestTransactions, TestSortedSetDocValues, TestBKD, TestBoostQuery, TestRollback, TestStressIndexing, TestTimSorterWorstCase, TestPerSegmentDeletes, TestBooleanQueryVisitSubscorers, TestReqOptSumScorer, TestMultiDocValues, TestDirectoryReaderReopen, TestFuzzyQuery, TestIndexWriterExceptions2, TestBytesRefHash, TestIndependenceStandardized, TestNumericUtils, TestNativeFSLockFactory, TestDocumentsWriterStallControl, TestMultiTermsEnum, TestBinaryDocValuesUpdates, TestComplexExplanationsOfNonMatches, TestSpanExplanations, TestIntArrayDocIdSet, TestBytesStore, TestSynonymQuery, TestAllFilesDetectTruncation, TestFrozenBufferedUpdates, TestTotalHitCountCollector, TestLongsRef, TestDisjunctionScoreBlockBoundaryPropagator, TestDaciukMihovAutomatonBuilder, TestLucene50CompoundFormat, TestExceedMaxTermLength, TestBooleanSimilarity, TestQueryVisitor, TestPrefixQuery, TestDeletionPolicy, TestTopDocsMerge, TestMinShouldMatch2, TestReusableStringReader, TestIndexWriterDelete, TestSpanMultiTermQueryWrapper, TestFileSwitchDirectory, Test2BPostingsBytes, Test2BSortedDocValuesFixedSorted, Test2BSortedDocValuesOrds, Test2BTerms, TestCheckIndex, TestCodecHoldsOpenFiles, TestCodecs, TestConcurrentMergeScheduler, TestConsistentFieldNumbers, TestCrash, TestCrashCausesCorruptIndex, TestCustomNorms, TestCustomTermFreq, TestDocIDMerger, TestDocInverterPerFieldErrorInfo, TestDocValues, TestDocValuesFieldUpdates, TestDocValuesIndexing, TestDocsAndPositions, TestDocsWithFieldSet, TestDocumentWriter, TestDocumentsWriterDeleteQueue, TestFieldsReader, TestFilterCodecReader, TestFilterDirectoryReader, TestFilterLeafReader, TestFilterMergePolicy, TestFlex, TestFlushByRamOrCountsPolicy, TestForTooMuchCloning, TestForceMergeForever, TestIndexInput, TestIndexManyDocuments, TestIndexOptions, TestIndexReaderClose, TestIndexSorting, TestIndexTooManyDocs, TestIndexWriter, TestIndexWriterCommit, TestIndexWriterConfig, TestIndexWriterOnJRECrash, TestIndexWriterOnVMError, TestDoubleRangeFieldQueries]
[junit4] Completed [409/528 (1!)] on J0 in 136.96s, 7 tests, 1 error <<< FAILURES!
[...truncated 396 lines...]
[junit4] JVM J0: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/core/test/temp/junit4-J0-20191220_154259_8788462022269486629784.sysout
[junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/heapdumps/java_pid20293.hprof ...
[junit4] Heap dump file created [522488851 bytes in 2.500 secs]
[junit4] <<< JVM J0: EOF ----
[...truncated 24030 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/build.xml:662: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/build.xml:586: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid20293.hprof
Total time: 316 minutes 1 second
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-BadApples-NightlyTests-8.x - Build # 47 -
Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-8.x/47/
7 tests failed.
FAILED: org.apache.lucene.replicator.nrt.TestStressNRTReplication.test
Error Message:
Captured an uncaught exception in thread: Thread[id=19, name=restarter, state=RUNNABLE, group=TGRP-TestStressNRTReplication]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=19, name=restarter, state=RUNNABLE, group=TGRP-TestStressNRTReplication]
at __randomizedtesting.SeedInfo.seed([AB64A8554B0B5F2A:2330978FE5F732D2]:0)
Caused by: java.lang.RuntimeException: java.lang.AssertionError: newPrimary.initCommitVersion=3 is missing from versionToTransLogLocation: keys=[0, 65, 2, 67, 4, 37, 11, 43, 16, 48, 20, 56, 27, 31, 63]
at __randomizedtesting.SeedInfo.seed([AB64A8554B0B5F2A]:0)
at org.apache.lucene.replicator.nrt.TestStressNRTReplication$RestartThread.run(TestStressNRTReplication.java:885)
Caused by: java.lang.AssertionError: newPrimary.initCommitVersion=3 is missing from versionToTransLogLocation: keys=[0, 65, 2, 67, 4, 37, 11, 43, 16, 48, 20, 56, 27, 31, 63]
at org.apache.lucene.replicator.nrt.TestStressNRTReplication.startPrimary(TestStressNRTReplication.java:451)
at org.apache.lucene.replicator.nrt.TestStressNRTReplication.promoteReplica(TestStressNRTReplication.java:423)
at org.apache.lucene.replicator.nrt.TestStressNRTReplication.access$500(TestStressNRTReplication.java:117)
at org.apache.lucene.replicator.nrt.TestStressNRTReplication$RestartThread.run(TestStressNRTReplication.java:830)
FAILED: junit.framework.TestSuite.org.apache.lucene.replicator.nrt.TestStressNRTReplication
Error Message:
1 thread leaked from SUITE scope at org.apache.lucene.replicator.nrt.TestStressNRTReplication: 1) Thread[id=113, name=pump0, state=RUNNABLE, group=TGRP-TestStressNRTReplication] at java.io.FileInputStream.readBytes(Native Method) at java.io.FileInputStream.read(FileInputStream.java:255) at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) at java.io.BufferedInputStream.read(BufferedInputStream.java:345) at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284) at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326) at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178) at java.io.InputStreamReader.read(InputStreamReader.java:184) at java.io.BufferedReader.fill(BufferedReader.java:161) at java.io.BufferedReader.readLine(BufferedReader.java:324) at java.io.BufferedReader.readLine(BufferedReader.java:389) at org.apache.lucene.replicator.nrt.ThreadPumper$1.run(ThreadPumper.java:37)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.lucene.replicator.nrt.TestStressNRTReplication:
1) Thread[id=113, name=pump0, state=RUNNABLE, group=TGRP-TestStressNRTReplication]
at java.io.FileInputStream.readBytes(Native Method)
at java.io.FileInputStream.read(FileInputStream.java:255)
at java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
at java.io.BufferedInputStream.read(BufferedInputStream.java:345)
at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284)
at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326)
at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178)
at java.io.InputStreamReader.read(InputStreamReader.java:184)
at java.io.BufferedReader.fill(BufferedReader.java:161)
at java.io.BufferedReader.readLine(BufferedReader.java:324)
at java.io.BufferedReader.readLine(BufferedReader.java:389)
at org.apache.lucene.replicator.nrt.ThreadPumper$1.run(ThreadPumper.java:37)
at __randomizedtesting.SeedInfo.seed([AB64A8554B0B5F2A]:0)
FAILED: junit.framework.TestSuite.org.apache.lucene.replicator.nrt.TestStressNRTReplication
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=113, name=pump0, state=RUNNABLE, group=TGRP-TestStressNRTReplication] at java.io.FileInputStream.readBytes(Native Method) at java.io.FileInputStream.read(FileInputStream.java:255) at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) at java.io.BufferedInputStream.read(BufferedInputStream.java:345) at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284) at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326) at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178) at java.io.InputStreamReader.read(InputStreamReader.java:184) at java.io.BufferedReader.fill(BufferedReader.java:161) at java.io.BufferedReader.readLine(BufferedReader.java:324) at java.io.BufferedReader.readLine(BufferedReader.java:389) at org.apache.lucene.replicator.nrt.ThreadPumper$1.run(ThreadPumper.java:37)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=113, name=pump0, state=RUNNABLE, group=TGRP-TestStressNRTReplication]
at java.io.FileInputStream.readBytes(Native Method)
at java.io.FileInputStream.read(FileInputStream.java:255)
at java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
at java.io.BufferedInputStream.read(BufferedInputStream.java:345)
at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284)
at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326)
at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178)
at java.io.InputStreamReader.read(InputStreamReader.java:184)
at java.io.BufferedReader.fill(BufferedReader.java:161)
at java.io.BufferedReader.readLine(BufferedReader.java:324)
at java.io.BufferedReader.readLine(BufferedReader.java:389)
at org.apache.lucene.replicator.nrt.ThreadPumper$1.run(ThreadPumper.java:37)
at __randomizedtesting.SeedInfo.seed([AB64A8554B0B5F2A]:0)
FAILED: org.apache.lucene.document.TestXYMultiPolygonShapeQueries.testRandomBig
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
at __randomizedtesting.SeedInfo.seed([BBA82E75B688EE57:3CFF53FA27D192D7]:0)
at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:84)
at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:57)
at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:168)
at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:154)
at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:141)
at org.apache.lucene.util.bkd.OfflinePointWriter.append(OfflinePointWriter.java:56)
at org.apache.lucene.util.bkd.BKDWriter.add(BKDWriter.java:247)
at org.apache.lucene.index.RandomCodec$1$1$1.visit(RandomCodec.java:129)
at org.apache.lucene.codecs.PointsWriter$1$1$1.visit(PointsWriter.java:117)
at org.apache.lucene.index.AssertingLeafReader$AssertingIntersectVisitor.visit(AssertingLeafReader.java:1171)
at org.apache.lucene.util.bkd.BKDReader.visitCompressedDocValues(BKDReader.java:735)
at org.apache.lucene.util.bkd.BKDReader.visitDocValuesWithCardinality(BKDReader.java:684)
at org.apache.lucene.util.bkd.BKDReader.visitDocValues(BKDReader.java:590)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:790)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:506)
at org.apache.lucene.index.AssertingLeafReader$AssertingPointValues.intersect(AssertingLeafReader.java:1052)
at org.apache.lucene.codecs.PointsWriter$1$1.intersect(PointsWriter.java:105)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest
Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest: 1) Thread[id=1472, name=Session-HouseKeeper-4d27248, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=1465, name=qtp1815760705-1465, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=1568, name=Connector-Scheduler-6805c61e, state=WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=1471, name=qtp1815760705-1471, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=1468, name=qtp1815760705-1468, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=1466, name=qtp1815760705-1466-acceptor-0@6565cb26-ServerConnector@6805c61e{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:46340}, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=1470, name=qtp1815760705-1470, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=1469, name=qtp1815760705-1469, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=1464, name=qtp1815760705-1464, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=1467, name=qtp1815760705-1467, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest:
1) Thread[id=1472, name=Session-HouseKeeper-4d27248, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=1465, name=qtp1815760705-1465, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=1568, name=Connector-Scheduler-6805c61e, state=WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=1471, name=qtp1815760705-1471, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=1468, name=qtp1815760705-1468, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=1466, name=qtp1815760705-1466-acceptor-0@6565cb26-ServerConnector@6805c61e{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:46340}, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=1470, name=qtp1815760705-1470, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=1469, name=qtp1815760705-1469, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=1464, name=qtp1815760705-1464, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=1467, name=qtp1815760705-1467, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([97B926D1D108389F]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=1472, name=Session-HouseKeeper-4d27248, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=1465, name=qtp1815760705-1465, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=1568, name=Connector-Scheduler-6805c61e, state=WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=1471, name=qtp1815760705-1471, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=1468, name=qtp1815760705-1468, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=1466, name=qtp1815760705-1466, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=1470, name=qtp1815760705-1470, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=1469, name=qtp1815760705-1469, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=1464, name=qtp1815760705-1464, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=1467, name=qtp1815760705-1467, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=1472, name=Session-HouseKeeper-4d27248, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=1465, name=qtp1815760705-1465, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=1568, name=Connector-Scheduler-6805c61e, state=WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=1471, name=qtp1815760705-1471, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=1468, name=qtp1815760705-1468, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=1466, name=qtp1815760705-1466, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=1470, name=qtp1815760705-1470, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=1469, name=qtp1815760705-1469, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=1464, name=qtp1815760705-1464, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=1467, name=qtp1815760705-1467, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([97B926D1D108389F]:0)
FAILED: org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey
Error Message:
Address already in use
Stack Trace:
java.net.BindException: Address already in use
at __randomizedtesting.SeedInfo.seed([97B926D1D108389F:1C9EF500900E931B]:0)
at sun.nio.ch.Net.bind0(Native Method)
at sun.nio.ch.Net.bind(Net.java:433)
at sun.nio.ch.Net.bind(Net.java:425)
at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
at org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:342)
at org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:308)
at org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
at org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:236)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.eclipse.jetty.server.Server.doStart(Server.java:396)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:567)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:506)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:474)
at org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:499)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11019 lines...]
[junit4] Suite: org.apache.lucene.replicator.nrt.TestStressNRTReplication
[junit4] 1> 0.001s : parent [TEST-TestStressNRTReplication.test-seed#[AB64A8554B0B5F2A]] change thread name from TEST-TestStressNRTReplication.test-seed#[AB64A8554B0B5F2A]
[junit4] 1> TEST: using 10 nodes
[junit4] 1> TEST: launch 2 indexer threads
[junit4] 1> TEST: launch 2 searcher threads
[junit4] 1> 0.072s : parent [ indexer0] top: indexer: updatePct=0.5014304102583915 sleepChance=59
[junit4] 1> 0.077s : parent [ indexer1] top: indexer: updatePct=0.5014304102583915 sleepChance=59
[junit4] 1> TEST: will run for 448 sec
[junit4] 1> 0.405s : parent [ restarter] node status x0(0.4s) x1(0.4s) x2(0.4s) x3(0.4s) x4(0.4s) x5(0.4s) x6(0.4s) x7(0.4s) x8(0.4s) x9(0.4s)
[junit4] 1> 0.405s : parent [ restarter] downNodes=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 0.406s : parent [ restarter] top: primary crashed; now pick replica to promote
[junit4] 1> 0.406s : parent [ restarter] top: no replicas running; skipping primary promotion
[junit4] 1> 0.406s : parent [ restarter] N0: top: cold start as primary
[junit4] 1> 0.406s : parent [ restarter] 0: top: startPrimary lastPrimaryVersion=0
[junit4] 1> 0.550s : parent [ restarter] JUnit version 4.12
[junit4] 1> 0.851s : parent [ main] PG=X 0 (of 10) nodes running:
[junit4] 1> 1.065s : parent [ main] PG=X 0 (of 10) nodes running:
[junit4] 1> 2.157s : parent [ restarter] .
[junit4] 1> 2.158s : parent [ main] PG=X 0 (of 10) nodes running:
[junit4] 1> 2.171s : parent [ restarter] PORT: 32834
[junit4] 1> 2.224s 0.1s: N0 [main child 0] top: corrupt unknown files
[junit4] 1> 2.233s : parent [ restarter] MDW: corrupt unknown files
[junit4] 1> 2.413s 0.3s: idle P0 [main child 0] top: now init primary
[junit4] 1> 2.421s 0.3s: idle P0 [main child 0] IWC:
[junit4] 1> 2.422s : parent [ restarter] analyzer=org.apache.lucene.analysis.MockAnalyzer
[junit4] 1> 2.425s : parent [ restarter] ramBufferSizeMB=16.0
[junit4] 1> 2.425s : parent [ restarter] maxBufferedDocs=-1
[junit4] 1> 2.425s : parent [ restarter] mergedSegmentWarmer=org.apache.lucene.replicator.nrt.PreCopyMergedSegmentWarmer@7a1f25a6
[junit4] 1> 2.426s : parent [ restarter] delPolicy=org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy
[junit4] 1> 2.426s : parent [ restarter] commit=null
[junit4] 1> 2.426s : parent [ restarter] openMode=CREATE_OR_APPEND
[junit4] 1> 2.426s : parent [ restarter] similarity=org.apache.lucene.search.similarities.AssertingSimilarity
[junit4] 1> 2.426s : parent [ restarter] mergeScheduler=ConcurrentMergeScheduler: maxThreadCount=2, maxMergeCount=3, ioThrottle=true
[junit4] 1> 2.426s : parent [ restarter] codec=Asserting(Lucene84): {}, docValues:{}, maxPointsInLeafNode=39, maxMBSortInHeap=6.153388661207469
[junit4] 1> 2.427s : parent [ restarter] infoStream=org.apache.lucene.util.InfoStream$NoOutput
[junit4] 1> 2.427s : parent [ restarter] mergePolicy=[LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=3, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 1> 2.427s : parent [ restarter] indexerThreadPool=org.apache.lucene.index.DocumentsWriterPerThreadPool@55e7beb7
[junit4] 1> 2.427s : parent [ restarter] readerPooling=true
[junit4] 1> 2.427s : parent [ restarter] perThreadHardLimitMB=1945
[junit4] 1> 2.427s : parent [ restarter] useCompoundFile=true
[junit4] 1> 2.427s : parent [ restarter] commitOnClose=true
[junit4] 1> 2.428s : parent [ restarter] indexSort=null
[junit4] 1> 2.428s : parent [ restarter] checkPendingFlushOnUpdate=true
[junit4] 1> 2.428s : parent [ restarter] softDeletesField=null
[junit4] 1> 2.428s : parent [ restarter] readerAttributes={}
[junit4] 1> 2.428s : parent [ restarter] writer=org.apache.lucene.index.IndexWriter@29de0dd6
[junit4] 1> 2.441s : parent [ restarter]
[junit4] 1> 2.442s 0.3s: idle P0 [main child 0] dir:
[junit4] 1> 2.447s : parent [ restarter] MockDirectoryWrapper(NIOFSDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index0-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@31ee3c4e)
[junit4] 1> 2.448s 0.3s: idle P0 [main child 0] commitData: []
[junit4] 1> 2.449s 0.3s: idle P0 [main child 0] add initial commitData version=0
[junit4] 1> 2.451s 0.3s: idle P0 [main child 0] now forcePrimaryVersion to version=1
[junit4] 1> 2.477s 0.3s: idle P0 [main child 0] top: switch to infos=segments: version=2
[junit4] 1> 2.494s 0.3s: idle P0 [main child 0] top: set copyState primaryGen=0 version=2 files=[]
[junit4] 1> 2.495s 0.3s: idle P0 [main child 0] init: infos version=2
[junit4] 1> 2.508s : parent [ restarter]
[junit4] 1> 2.509s : parent [ restarter] COMMIT VERSION: 0
[junit4] 1> 2.509s : parent [ restarter]
[junit4] 1> 2.510s : parent [ restarter] INFOS VERSION: 2
[junit4] 1> 2.510s 0.4s: idle P0 [main child 0] top: will crash after 40.998 seconds
[junit4] 1> 2.524s : parent [ restarter]
[junit4] 1> 2.524s : parent [ restarter] NODE STARTED
[junit4] 1> 2.535s : parent [ restarter] top: node=0 started at tcpPort=32834 initCommitVersion=0 initInfosVersion=2
[junit4] 1> 2.536s : parent [ restarter] top: record transLogLoc: version=2 loc=0
[junit4] 1> 2.536s : parent [ restarter] top: record marker count: version=2 count=0
[junit4] 1> 2.537s : parent [ restarter] top: now change lastPrimaryVersion from 0 to 2; startup marker count 0
[junit4] 1> 2.537s : parent [ restarter] top: start translog replay 0 (version=0) to 0 (translog end)
[junit4] 1> 2.588s : parent [ restarter] top: done translog replay; took 51.620597 msec; now publish primary
[junit4] 1> 2.589s : parent [ restarter] top: send 0 replicas to primary
[junit4] 1> 2.588s 0.4s: idle P0 [ indexing] start handling indexing socket=Socket[addr=/127.0.0.1,port=57106,localport=32834]
[junit4] 1> 2.590s : parent [ main] PG=0 1 (of 10) nodes running: P0
[junit4] 1> 2.590s : parent [ main] top: now commit node=P0 tcpPort=32834
[junit4] 1> 2.632s 0.5s: idle P0 [ set repls] top: set replicasIDs=[] tcpPorts=[]
[junit4] 1> 2.633s : parent [Searcher node=P0 tcpPort=32834] top: searcher: record search hitCount version=2 hitCount=0 node=P0 tcpPort=32834
[junit4] 1> 2.633s 0.5s: idle P0 [ commit] top: commit commitData={__version=2, __primaryGen=0}
[junit4] 1> 2.636s 0.5s: idle P0 [ indexing] start handling indexing socket=Socket[addr=/127.0.0.1,port=57114,localport=32834]
[junit4] 1> 2.644s 0.5s: idle P0 [ indexing] start handling indexing socket=Socket[addr=/127.0.0.1,port=57116,localport=32834]
[junit4] 1> 2.967s : parent [ restarter] node status p0(2.6s) x1(3.0s) x2(3.0s) x3(3.0s) x4(3.0s) x5(3.0s) x6(3.0s) x7(3.0s) x8(3.0s) x9(3.0s)
[junit4] 1> 2.967s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 2.967s : parent [ restarter] N1: top: start up: launch thread
[junit4] 1> 2.997s : parent [ start R1] N1: top: start up thread
[junit4] 1> 3.243s : parent [ start R1] JUnit version 4.12
[junit4] 1> 3.295s : parent [ restarter] node status p0(2.9s) s1(0.3s) x2(3.3s) x3(3.3s) x4(3.3s) x5(3.3s) x6(3.3s) x7(3.3s) x8(3.3s) x9(3.3s)
[junit4] 1> 3.295s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 3.295s : parent [ restarter] N6: top: start up: launch thread
[junit4] 1> 3.306s : parent [ start R6] N6: top: start up thread
[junit4] 1> Indexer: now pause for 1826 msec...
[junit4] 1> 3.508s : parent [ restarter] node status p0(3.1s) s1(0.5s) x2(3.5s) x3(3.5s) x4(3.5s) x5(3.5s) s6(0.2s) x7(3.5s) x8(3.5s) x9(3.5s)
[junit4] 1> 3.508s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 3.509s : parent [ restarter] N9: top: start up: launch thread
[junit4] 1> Indexer: now pause for 1826 msec...
[junit4] 1> 3.533s : parent [ start R9] N9: top: start up thread
[junit4] 1> 3.620s : parent [ restarter] node status p0(3.2s) s1(0.6s) x2(3.6s) x3(3.6s) x4(3.6s) x5(3.6s) s6(0.3s) x7(3.6s) x8(3.6s) s9(0.1s)
[junit4] 1> 3.621s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 3.621s : parent [ restarter] N7: top: start up: launch thread
[junit4] 1> 3.631s : parent [ start R7] N7: top: start up thread
[junit4] 1> 3.681s : parent [ start R6] JUnit version 4.12
[junit4] 1> 3.791s : parent [ start R9] JUnit version 4.12
[junit4] 1> 3.998s : parent [ start R7] JUnit version 4.12
[junit4] 1> 4.086s : parent [ restarter] node status p0(3.7s) s1(1.1s) x2(4.1s) x3(4.1s) x4(4.1s) x5(4.1s) s6(0.8s) s7(0.5s) x8(4.1s) s9(0.6s)
[junit4] 1> 4.086s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 4.086s : parent [ restarter] node 1 still starting
[junit4] 1> 4.269s : parent [ main] PG=0 1 (of 10) nodes running: P0
[junit4] 1> 4.322s : parent [ restarter] node status p0(3.9s) s1(1.3s) x2(4.3s) x3(4.3s) x4(4.3s) x5(4.3s) s6(1.0s) s7(0.7s) x8(4.3s) s9(0.8s)
[junit4] 1> 4.322s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 4.322s : parent [ restarter] N2: top: start up: launch thread
[junit4] 1> 4.332s : parent [ start R2] N2: top: start up thread
[junit4] 1> 4.399s : parent [ restarter] node status p0(4.0s) s1(1.4s) s2(0.1s) x3(4.4s) x4(4.4s) x5(4.4s) s6(1.1s) s7(0.8s) x8(4.4s) s9(0.9s)
[junit4] 1> 4.399s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 4.399s : parent [ restarter] N4: top: start up: launch thread
[junit4] 1> 4.429s : parent [ start R4] N4: top: start up thread
[junit4] 1> 4.601s : parent [ restarter] node status p0(4.2s) s1(1.6s) s2(0.3s) x3(4.6s) s4(0.2s) x5(4.6s) s6(1.3s) s7(1.0s) x8(4.6s) s9(1.1s)
[junit4] 1> 4.601s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 4.601s : parent [ restarter] N5: top: start up: launch thread
[junit4] 1> 4.623s : parent [ start R5] N5: top: start up thread
[junit4] 1> 4.748s : parent [ restarter] node status p0(4.3s) s1(1.7s) s2(0.4s) x3(4.7s) s4(0.3s) s5(0.1s) s6(1.4s) s7(1.1s) x8(4.7s) s9(1.2s)
[junit4] 1> 4.748s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 4.748s : parent [ restarter] N8: top: start up: launch thread
[junit4] 1> 4.755s : parent [ start R8] N8: top: start up thread
[junit4] 1> 4.782s : parent [ start R5] JUnit version 4.12
[junit4] 1> 4.811s : parent [ restarter] top: full cluster crash
[junit4] 1> 4.812s : parent [ restarter] top: N0: top: now crash node
[junit4] 1> 4.852s : parent [ start R2] JUnit version 4.12
[junit4] 1> 5.057s : parent [ start R4] JUnit version 4.12
[junit4] 1> 5.182s : parent [ pump0] now wait for process java.lang.UNIXProcess@54064533
[junit4] 1> 5.186s : parent [ pump0] done wait for process java.lang.UNIXProcess@54064533
[junit4] 1> 5.186s : parent [ pump0] exit value=143 willCrash=true
[junit4] 1> 5.187s : parent [ pump0] top: P0 tcpPort=32834: primary process finished
[junit4] 1> 5.232s : parent [ restarter] top: N0: top: done crash node
[junit4] 1> 5.233s : parent [ restarter] N1: top: wait for startup so we can crash...
[junit4] 1> 5.257s : parent [ start R1] .
[junit4] 1> 5.258s : parent [ start R1] PORT: 37103
[junit4] 1> Indexer: done pause for a bit...
[junit4] 1> 5.272s : parent [ indexer p0] top: indexer lost connection to primary
[junit4] 1> 5.324s : parent [ start R8] JUnit version 4.12
[junit4] 1> Indexer: done pause for a bit...
[junit4] 1> 5.356s : parent [ indexer p0] top: indexer lost connection to primary
[junit4] 1> 5.392s 0.1s: N1 [main child 1] top: corrupt unknown files
[junit4] 1> 5.396s : parent [ start R1] MDW: corrupt unknown files
[junit4] 1> 5.424s 0.2s: idle R1 [main child 1] top: init replica dir=MockDirectoryWrapper(MMapDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index1-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@58f4b779)
[junit4] 1> 5.436s 0.2s: init R1 [main child 1] top: will rate limit file fetch to 5.42 MB/sec
[junit4] 1> 5.436s 0.2s: init R1 [main child 1] top: now start
[junit4] 1> 5.438s 0.2s: init R1 [main child 1] top: init: no segments in index
[junit4] 1> 5.452s 0.2s: init R1 [main child 1] top: delete unknown files on init: all files=[write.lock]
[junit4] 1> 5.461s 0.2s: init R1 [main child 1] now delete 0 files: []
[junit4] 1> 5.464s 0.2s: init R1 [main child 1] top: done delete unknown files on init: all files=[write.lock]
[junit4] 1> 5.464s 0.2s: init R1 [main child 1] top: myPrimaryGen=-1
[junit4] 1> 5.470s 0.2s: init R1 [main child 1] top: same primary as before
[junit4] 1> 5.471s 0.2s: init R1 [main child 1] send new_replica to primary tcpPort=32834
[junit4] 1> 5.488s 0.2s: init R1 [main child 1] ignoring exc java.net.ConnectException: Connection refused (Connection refused) sending new_replica to primary tcpPort=32834
[junit4] 1> 5.489s 0.2s: init R1 [main child 1] SegmentInfosSearcherManager.init: use incoming infos=segments:
[junit4] 1> 5.585s 0.3s: init R1 [main child 1] top: done start
[junit4] 1> 5.596s : parent [ main] PG=X 0 (of 10) nodes running:
[junit4] 1> 5.604s : parent [ start R1]
[junit4] 1> 5.604s : parent [ start R1] INFOS VERSION: 0
[junit4] 1> 5.604s 0.4s: idle R1 [main child 1] top: will crash after 18.191 seconds
[junit4] 1> 5.627s : parent [ start R1]
[junit4] 1> 5.627s : parent [ start R1] NODE STARTED
[junit4] 1> 5.655s : parent [ start R1] top: node=1 started at tcpPort=37103 initCommitVersion=-1 initInfosVersion=0
[junit4] 1> 5.693s : parent [ restarter] N1: top: done wait for startup
[junit4] 1> 5.693s : parent [ restarter] top: N1: top: now crash node
[junit4] 1> 5.944s : parent [ main] PG=X 1 (of 10) nodes running: R1
[junit4] 1> 6.101s : parent [ pump1] now wait for process java.lang.UNIXProcess@35ff0009
[junit4] 1> 6.102s : parent [ pump1] done wait for process java.lang.UNIXProcess@35ff0009
[junit4] 1> 6.102s : parent [ pump1] exit value=143 willCrash=true
[junit4] 1> 6.103s : parent [ pump1] top: R1 tcpPort=37103: replica process finished
[junit4] 1> 6.124s : parent [ restarter] top: N1: top: done crash node
[junit4] 1> 6.124s : parent [ restarter] N2: top: wait for startup so we can crash...
[junit4] 1> 6.420s : parent [ start R6] .
[junit4] 1> 6.420s : parent [ start R6] PORT: 45805
[junit4] 1> 6.444s : parent [ main] PG=X 0 (of 10) nodes running:
[junit4] 1> 6.560s 0.2s: N6 [main child 6] top: corrupt unknown files
[junit4] 1> 6.576s : parent [ start R6] MDW: corrupt unknown files
[junit4] 1> 6.581s 0.2s: idle R6 [main child 6] top: init replica dir=MockDirectoryWrapper(MMapDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index6-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7b556f99)
[junit4] 1> 6.608s 0.2s: init R6 [main child 6] top: will rate limit file fetch to 9.92 MB/sec
[junit4] 1> 6.609s 0.2s: init R6 [main child 6] top: now start
[junit4] 1> 6.632s 0.2s: init R6 [main child 6] top: init: no segments in index
[junit4] 1> 6.634s 0.2s: init R6 [main child 6] top: delete unknown files on init: all files=[write.lock]
[junit4] 1> 6.636s 0.2s: init R6 [main child 6] now delete 0 files: []
[junit4] 1> 6.644s 0.2s: init R6 [main child 6] top: done delete unknown files on init: all files=[write.lock]
[junit4] 1> 6.644s 0.2s: init R6 [main child 6] top: myPrimaryGen=-1
[junit4] 1> 6.645s 0.2s: init R6 [main child 6] top: same primary as before
[junit4] 1> 6.649s 0.3s: init R6 [main child 6] send new_replica to primary tcpPort=32834
[junit4] 1> 6.655s 0.3s: init R6 [main child 6] ignoring exc java.net.ConnectException: Connection refused (Connection refused) sending new_replica to primary tcpPort=32834
[junit4] 1> 6.666s 0.3s: init R6 [main child 6] SegmentInfosSearcherManager.init: use incoming infos=segments:
[junit4] 1> 6.770s 0.4s: init R6 [main child 6] top: done start
[junit4] 1> 6.786s : parent [ start R6]
[junit4] 1> 6.787s : parent [ start R6] INFOS VERSION: 0
[junit4] 1> 6.788s 0.4s: idle R6 [main child 6] top: will close after 25.982 seconds
[junit4] 1> 6.798s : parent [ start R6]
[junit4] 1> 6.800s : parent [ start R6] NODE STARTED
[junit4] 1> 6.800s : parent [ start R6] top: node=6 started at tcpPort=45805 initCommitVersion=-1 initInfosVersion=0
[junit4] 1> 6.849s : parent [Searcher node=R6 tcpPort=45805] top: searcher: record search hitCount version=0 hitCount=0 node=R6 tcpPort=45805
[junit4] 1> 7.166s : parent [ start R7] .
[junit4] 1> 7.177s : parent [ start R7] PORT: 46221
[junit4] 1> 7.193s : parent [ start R9] .
[junit4] 1> 7.193s : parent [ start R9] PORT: 37069
[junit4] 1> 7.238s 0.1s: N7 [main child 7] top: corrupt unknown files
[junit4] 1> 7.242s : parent [ start R7] MDW: corrupt unknown files
[junit4] 1> 7.252s 0.1s: idle R7 [main child 7] top: init replica dir=MockDirectoryWrapper(SimpleFSDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index7-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@5970f72d)
[junit4] 1> 7.244s 0.1s: N9 [main child 9] top: corrupt unknown files
[junit4] 1> 7.262s : parent [ start R9] MDW: corrupt unknown files
[junit4] 1> 7.277s 0.1s: idle R9 [main child 9] top: init replica dir=MockDirectoryWrapper(SimpleFSDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index9-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@5970f72d)
[junit4] 1> 7.279s 0.1s: init R7 [main child 7] top: will rate limit file fetch to 15.24 MB/sec
[junit4] 1> 7.280s 0.1s: init R7 [main child 7] top: now start
[junit4] 1> 7.305s 0.1s: init R9 [main child 9] top: will rate limit file fetch to 19.85 MB/sec
[junit4] 1> 7.305s 0.1s: init R9 [main child 9] top: now start
[junit4] 1> 7.308s 0.1s: init R7 [main child 7] top: init: no segments in index
[junit4] 1> 7.309s 0.1s: init R7 [main child 7] top: delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.311s 0.1s: init R7 [main child 7] now delete 0 files: []
[junit4] 1> 7.312s 0.2s: init R7 [main child 7] top: done delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.313s 0.2s: init R7 [main child 7] top: myPrimaryGen=-1
[junit4] 1> 7.314s 0.2s: init R7 [main child 7] top: same primary as before
[junit4] 1> 7.314s 0.2s: init R7 [main child 7] send new_replica to primary tcpPort=32834
[junit4] 1> 7.325s 0.2s: init R7 [main child 7] ignoring exc java.net.ConnectException: Connection refused (Connection refused) sending new_replica to primary tcpPort=32834
[junit4] 1> 7.333s 0.2s: init R9 [main child 9] top: init: no segments in index
[junit4] 1> 7.333s 0.2s: init R7 [main child 7] SegmentInfosSearcherManager.init: use incoming infos=segments:
[junit4] 1> 7.336s 0.2s: init R9 [main child 9] top: delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.352s 0.2s: init R9 [main child 9] now delete 0 files: []
[junit4] 1> 7.353s 0.2s: init R9 [main child 9] top: done delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.354s 0.2s: init R9 [main child 9] top: myPrimaryGen=-1
[junit4] 1> 7.355s 0.2s: init R9 [main child 9] top: same primary as before
[junit4] 1> 7.355s 0.2s: init R9 [main child 9] send new_replica to primary tcpPort=32834
[junit4] 1> 7.364s 0.2s: init R9 [main child 9] ignoring exc java.net.ConnectException: Connection refused (Connection refused) sending new_replica to primary tcpPort=32834
[junit4] 1> 7.365s 0.2s: init R9 [main child 9] SegmentInfosSearcherManager.init: use incoming infos=segments:
[junit4] 1> 7.425s 0.3s: init R7 [main child 7] top: done start
[junit4] 1> 7.436s : parent [ start R7]
[junit4] 1> 7.436s : parent [ start R7] INFOS VERSION: 0
[junit4] 1> 7.436s 0.3s: idle R7 [main child 7] top: will close after 45.189 seconds
[junit4] 1> 7.419s 0.2s: init R9 [main child 9] top: done start
[junit4] 1> 7.450s : parent [ start R7]
[junit4] 1> 7.450s : parent [ start R7] NODE STARTED
[junit4] 1> 7.452s : parent [ start R9]
[junit4] 1> 7.452s : parent [ start R9] INFOS VERSION: 0
[junit4] 1> 7.452s 0.3s: idle R9 [main child 9] top: will crash after 32.865 seconds
[junit4] 1> 7.460s : parent [ start R9]
[junit4] 1> 7.460s : parent [ start R9] NODE STARTED
[junit4] 1> 7.460s : parent [ start R7] top: node=7 started at tcpPort=46221 initCommitVersion=-1 initInfosVersion=0
[junit4] 1> 7.464s : parent [ start R9] top: node=9 started at tcpPort=37069 initCommitVersion=-1 initInfosVersion=0
[junit4] 1> 7.622s : parent [ start R2] .
[junit4] 1> 7.622s : parent [ start R2] PORT: 36326
[junit4] 1> 7.671s : parent [ start R5] .
[junit4] 1> 7.677s : parent [ start R5] PORT: 35263
[junit4] 1> 7.682s 0.1s: N2 [main child 2] top: corrupt unknown files
[junit4] 1> 7.690s : parent [ start R2] MDW: corrupt unknown files
[junit4] 1> 7.693s 0.1s: idle R2 [main child 2] top: init replica dir=MockDirectoryWrapper(MMapDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index2-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1934e9c8)
[junit4] 1> 7.708s 0.1s: init R2 [main child 2] top: will rate limit file fetch to 11.44 MB/sec
[junit4] 1> 7.708s 0.1s: init R2 [main child 2] top: now start
[junit4] 1> 7.718s 0.1s: init R2 [main child 2] top: init: no segments in index
[junit4] 1> 7.719s 0.1s: init R2 [main child 2] top: delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.720s 0.1s: init R2 [main child 2] now delete 0 files: []
[junit4] 1> 7.722s 0.1s: init R2 [main child 2] top: done delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.722s 0.1s: init R2 [main child 2] top: myPrimaryGen=-1
[junit4] 1> 7.723s 0.1s: init R2 [main child 2] top: same primary as before
[junit4] 1> 7.723s 0.1s: init R2 [main child 2] send new_replica to primary tcpPort=32834
[junit4] 1> 7.727s 0.1s: init R2 [main child 2] ignoring exc java.net.ConnectException: Connection refused (Connection refused) sending new_replica to primary tcpPort=32834
[junit4] 1> 7.728s 0.1s: init R2 [main child 2] SegmentInfosSearcherManager.init: use incoming infos=segments:
[junit4] 1> 7.774s : parent [ main] PG=X 3 (of 10) nodes running: R6 R7 R9
[junit4] 1> 7.775s 0.2s: init R2 [main child 2] top: done start
[junit4] 1> 7.784s : parent [ start R2]
[junit4] 1> 7.784s : parent [ start R2] INFOS VERSION: 0
[junit4] 1> 7.784s 0.2s: idle R2 [main child 2] top: will close after 56.721 seconds
[junit4] 1> 7.789s : parent [ start R2]
[junit4] 1> 7.790s : parent [ start R2] NODE STARTED
[junit4] 1> 7.791s : parent [ start R2] top: node=2 started at tcpPort=36326 initCommitVersion=-1 initInfosVersion=0
[junit4] 1> 7.782s 0.1s: N5 [main child 5] top: corrupt unknown files
[junit4] 1> 7.794s : parent [ start R5] MDW: corrupt unknown files
[junit4] 1> 7.795s : parent [ restarter] N2: top: done wait for startup
[junit4] 1> 7.795s : parent [ restarter] top: N2: top: now crash node
[junit4] 1> 7.806s 0.1s: idle R5 [main child 5] top: init replica dir=MockDirectoryWrapper(SimpleFSDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index5-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@5970f72d)
[junit4] 1> 7.822s 0.2s: init R5 [main child 5] top: will rate limit file fetch to 8.87 MB/sec
[junit4] 1> 7.824s 0.2s: init R5 [main child 5] top: now start
[junit4] 1> 7.854s 0.2s: init R5 [main child 5] top: init: no segments in index
[junit4] 1> 7.857s 0.2s: init R5 [main child 5] top: delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.858s 0.2s: init R5 [main child 5] now delete 0 files: []
[junit4] 1> 7.862s 0.2s: init R5 [main child 5] top: done delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.862s 0.2s: init R5 [main child 5] top: myPrimaryGen=-1
[junit4] 1> 7.862s 0.2s: init R5 [main child 5] top: same primary as before
[junit4] 1> 7.863s 0.2s: init R5 [main child 5] send new_replica to primary tcpPort=32834
[junit4] 1> 7.866s 0.2s: init R5 [main child 5] ignoring exc java.net.ConnectException: Connection refused (Connection refused) sending new_replica to primary tcpPort=32834
[junit4] 1> 7.867s 0.2s: init R5 [main child 5] SegmentInfosSearcherManager.init: use incoming infos=segments:
[junit4] 1> 7.869s : parent [ start R8] .
[junit4] 1> 7.869s : parent [ start R8] PORT: 32905
[junit4] 1> 7.908s : parent [ start R4] .
[junit4] 1> 7.909s : parent [ start R4] PORT: 46299
[junit4] 1> 7.915s 0.2s: init R5 [main child 5] top: done start
[junit4] 1> 7.943s 0.1s: N8 [main child 8] top: corrupt unknown files
[junit4] 1> 7.949s : parent [ start R8] MDW: corrupt unknown files
[junit4] 1> 7.956s 0.1s: idle R8 [main child 8] top: init replica dir=MockDirectoryWrapper(MMapDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index8-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@75ae4b61)
[junit4] 1> 7.960s : parent [ start R5]
[junit4] 1> 7.960s : parent [ start R5] INFOS VERSION: 0
[junit4] 1> 7.960s 0.3s: idle R5 [main child 5] top: will crash after 56.741 seconds
[junit4] 1> 7.960s 0.1s: N4 [main child 4] top: corrupt unknown files
[junit4] 1> 7.965s : parent [ start R4] MDW: corrupt unknown files
[junit4] 1> 7.974s : parent [ start R5]
[junit4] 1> 7.974s : parent [ start R5] NODE STARTED
[junit4] 1> 7.975s : parent [ start R5] top: node=5 started at tcpPort=35263 initCommitVersion=-1 initInfosVersion=0
[junit4] 1> 7.977s 0.1s: init R8 [main child 8] top: will rate limit file fetch to 17.82 MB/sec
[junit4] 1> 7.977s 0.1s: init R8 [main child 8] top: now start
[junit4] 1> 7.980s 0.1s: idle R4 [main child 4] top: init replica dir=MockDirectoryWrapper(SimpleFSDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index4-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@5970f72d)
[junit4] 1> 7.979s 0.1s: init R8 [main child 8] top: init: no segments in index
[junit4] 1> 7.981s 0.1s: init R8 [main child 8] top: delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.982s 0.1s: init R8 [main child 8] now delete 0 files: []
[junit4] 1> 7.987s 0.1s: init R8 [main child 8] top: done delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.988s 0.1s: init R8 [main child 8] top: myPrimaryGen=-1
[junit4] 1> 7.988s 0.1s: init R8 [main child 8] top: same primary as before
[junit4] 1> 7.989s 0.1s: init R8 [main child 8] send new_replica to primary tcpPort=32834
[junit4] 1> 7.993s 0.1s: init R8 [main child 8] ignoring exc java.net.ConnectException: Connection refused (Connection refused) sending new_replica to primary tcpPort=32834
[junit4] 1> 7.994s 0.1s: init R4 [main child 4] top: will rate limit file fetch to 9.69 MB/sec
[junit4] 1> 7.994s 0.1s: init R4 [main child 4] top: now start
[junit4] 1> 7.994s 0.1s: init R8 [main child 8] SegmentInfosSearcherManager.init: use incoming infos=segments:
[junit4] 1> 8.000s 0.1s: init R4 [main child 4] top: init: no segments in index
[junit4] 1> 8.001s 0.1s: init R4 [main child 4] top: delete unknown files on init: all files=[write.lock]
[junit4] 1> 8.002s 0.1s: init R4 [main child 4] now delete 0 files: []
[junit4] 1> 8.004s 0.1s: init R4 [main child 4] top: done delete unknown files on init: all files=[write.lock]
[junit4] 1> 8.004s 0.1s: init R4 [main child 4] top: myPrimaryGen=-1
[junit4] 1> 8.004s 0.1s: init R4 [main child 4] top: same primary as before
[junit4] 1> 8.005s 0.1s: init R4 [main child 4] send new_replica to primary tcpPort=32834
[junit4] 1> 8.012s 0.1s: init R4 [main child 4] ignoring exc java.net.ConnectException: Connection refused (Connection refused) sending new_replica to primary tcpPort=32834
[junit4] 1> 8.013s 0.1s: init R4 [main child 4] SegmentInfosSearcherManager.init: use incoming infos=segments:
[junit4] 1> 8.032s 0.2s: init R8 [main child 8] top: done start
[junit4] 1> 8.034s : parent [ start R8]
[junit4] 1> 8.034s : parent [ start R8] INFOS VERSION: 0
[junit4] 1> 8.035s 0.2s: idle R8 [main child 8] top: will crash after 35.791 seconds
[junit4] 1> 8.036s : parent [ start R8]
[junit4] 1> 8.036s : parent [ start R8] NODE STARTED
[junit4] 1> 8.037s : parent [ start R8] top: node=8 started at tcpPort=32905 initCommitVersion=-1 initInfosVersion=0
[junit4] 1> 8.054s 0.1s: init R4 [main child 4] top: done start
[junit4] 1> 8.092s : parent [ start R4]
[junit4] 1> 8.092s : parent [ start R4] INFOS VERSION: 0
[junit4] 1> 8.092s 0.2s: idle R4 [main child 4] top: will crash after 51.743 seconds
[junit4] 1> 8.093s : parent [ start R4]
[junit4] 1> 8.093s : parent [ start R4] NODE STARTED
[junit4] 1> 8.095s : parent [ start R4] top: node=4 started at tcpPort=46299 initCommitVersion=-1 initInfosVersion=0
[junit4] 1> 8.120s : parent [ main] PG=X 7 (of 10) nodes running: R2 R4 R5 R6 R7 R8 R9
[junit4] 1> 8.155s : parent [ pump2] now wait for process java.lang.UNIXProcess@7d80c82
[junit4] 1> 8.158s : parent [ pump2] done wait for process java.lang.UNIXProcess@7d80c82
[junit4] 1> 8.158s : parent [ pump2] exit value=143 willCrash=false
[junit4] 1> 8.159s : parent [ pump2] top: R2 tcpPort=36326: replica process finished
[junit4] 1> 8.159s : parent [ restarter] top: N2: top: done crash node
[junit4] 1> 8.159s : parent [ restarter] top: N4: top: now crash node
[junit4] 1> 8.502s : parent [ pump4] now wait for process java.lang.UNIXProcess@5c539721
[junit4] 1> 8.503s : parent [ pump4] done wait for process java.lang.UNIXProcess@5c539721
[junit4] 1> 8.503s : parent [ pump4] exit value=143 willCrash=true
[junit4] 1> 8.504s : parent [ pump4] top: R4 tcpPort=46299: replica process finished
[junit4] 1> 8.504s : parent [ restarter] top: N4: top: done crash node
[junit4] 1> 8.504s : parent [ restarter] top: N5: top: now crash node
[junit4] 1> 8.564s : parent [ main] PG=X 5 (of 10) nodes running: R5 R6 R7 R8 R9
[junit4] 1> 8.879s : parent [ pump5] now wait for process java.lang.UNIXProcess@22d9f740
[junit4] 1> 8.881s : parent [ pump5] done wait for process java.lang.UNIXProcess@22d9f740
[junit4] 1> 8.881s : parent [ pump5] exit value=143 willCrash=true
[junit4] 1> 8.881s : parent [ pump5] top: R5 tcpPort=35263: replica process finished
[junit4] 1> 8.881s : parent [ restarter] top: N5: top: done crash node
[junit4] 1> 8.881s : parent [ restarter] top: N6: top: now crash node
[junit4] 1> 9.230s : parent [ pump6] now wait for process java.lang.UNIXProcess@16bd6fed
[junit4] 1> 9.232s : parent [ pump6] done wait for process java.lang.UNIXProcess@16bd6fed
[junit4] 1> 9.232s : parent [ pump6] exit value=143 willCrash=false
[junit4] 1> 9.232s : parent [ pump6] top: R6 tcpPort=45805: replica process finished
[junit4] 1> 9.232s : parent [ restarter] top: N6: top: done crash node
[junit4] 1> 9.233s : parent [ restarter] top: N7: top: now crash node
[junit4] 1> 9.445s : parent [ main] PG=X 3 (of 10) nodes running: R7 R8 R9
[junit4] 1> 9.568s : parent [ pump7] now wait for process java.lang.UNIXProcess@52fadad
[junit4] 1> 9.569s : parent [ pump7] done wait for process java.lang.UNIXProcess@52fadad
[junit4] 1> 9.569s : parent [ pump7] exit value=143 willCrash=false
[junit4] 1> 9.569s : parent [ pump7] top: R7 tcpPort=46221: replica process finished
[junit4] 1> 9.570s : parent [ restarter] top: N7: top: done crash node
[junit4] 1> 9.570s : parent [ restarter] top: N8: top: now crash node
[junit4] 1> 9.898s : parent [ pump8] now wait for process java.lang.UNIXProcess@36329d4b
[junit4] 1> 9.906s : parent [ pump8] done wait for process java.lang.UNIXProcess@36329d4b
[junit4] 1> 9.906s : parent [ pump8] exit value=143 willCrash=true
[junit4] 1> 9.906s : parent [ pump8] top: R8 tcpPort=32905: replica process finished
[junit4] 1> 9.906s : parent [ restarter] top: N8: top: done crash node
[junit4] 1> 9.906s : parent [ restarter] top: N9: top: now crash node
[junit4] 1> 10.245s : parent [ pump9] now wait for process java.lang.UNIXProcess@e945446
[junit4] 1> 10.253s : parent [ pump9] done wait for process java.lang.UNIXProcess@e945446
[junit4] 1> 10.253s : parent [ pump9] exit value=143 willCrash=true
[junit4] 1> 10.253s : parent [ pump9] top: R9 tcpPort=37069: replica process finished
[junit4] 1> 10.254s : parent [ restarter] top: N9: top: done crash node
[junit4] 1> 10.254s : parent [ restarter] node status x0(5.1s) x1(4.2s) x2(2.1s) x3(10.3s) x4(1.8s) x5(1.4s) x6(1.0s) x7(0.7s) x8(0.3s) x9(0.0s)
[junit4] 1> 10.254s : parent [ restarter] downNodes=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 10.254s : parent [ restarter] top: primary crashed; now pick replica to promote
[junit4] 1> 10.254s : parent [ restarter] top: no replicas running; skipping primary promotion
[junit4] 1> 10.254s : parent [ restarter] N0: top: cold start as primary
[junit4] 1> 10.254s : parent [ restarter] 0: top: startPrimary lastPrimaryVersion=2
[junit4] 1> 10.483s : parent [ restarter] JUnit version 4.12
[junit4] 1> 10.903s : parent [ main] PG=X 0 (of 10) nodes running:
[junit4] 1> 11.695
[...truncated too long message...]
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 3) Thread[id=1568, name=Connector-Scheduler-6805c61e, state=WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 4) Thread[id=1471, name=qtp1815760705-1471, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=1468, name=qtp1815760705-1468, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=1466, name=qtp1815760705-1466, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 7) Thread[id=1470, name=qtp1815760705-1470, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 8) Thread[id=1469, name=qtp1815760705-1469, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 9) Thread[id=1464, name=qtp1815760705-1464, state=RUNNABLE, group=TGRP-ShardSplitTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 10) Thread[id=1467, name=qtp1815760705-1467, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([97B926D1D108389F]:0)
[junit4] Completed [25/889 (1!)] on J1 in 914.68s, 11 tests, 3 errors <<< FAILURES!
[...truncated 12197 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/build.xml:662: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/build.xml:586: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid22088.hprof
Total time: 438 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-BadApples-NightlyTests-8.x - Build # 46 -
Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-8.x/46/
11 tests failed.
FAILED: org.apache.lucene.codecs.asserting.TestAssertingDocValuesFormat.testSparseLongNumericsVsStoredFields
Error Message:
Test abandoned because suite timeout was reached.
Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
at __randomizedtesting.SeedInfo.seed([D446FDC9F8B4342D]:0)
FAILED: junit.framework.TestSuite.org.apache.lucene.codecs.asserting.TestAssertingDocValuesFormat
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([D446FDC9F8B4342D]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest
Error Message:
access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
at __randomizedtesting.SeedInfo.seed([D1C21E02EF7F767C]:0)
at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
at java.security.AccessController.checkPermission(AccessController.java:884)
at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
at org.apache.hadoop.util.Shell.run(Shell.java:901)
at org.apache.hadoop.fs.DF.getFilesystem(DF.java:74)
at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker$CheckedVolume.<init>(NameNodeResourceChecker.java:70)
at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.addDirToCheck(NameNodeResourceChecker.java:166)
at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.<init>(NameNodeResourceChecker.java:135)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startCommonServices(FSNamesystem.java:1168)
at org.apache.hadoop.hdfs.server.namenode.NameNode.startCommonServices(NameNode.java:791)
at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:717)
at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:940)
at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:913)
at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1646)
at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNode(MiniDFSCluster.java:1314)
at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1083)
at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
at org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.setupClass(HdfsUnloadDistributedZkTest.java:39)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest
Error Message:
15 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest: 1) Thread[id=3399, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=3406, name=qtp1114988144-3406, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=3417, name=IPC Server idle connection scanner for port 38153, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) at java.util.TimerThread.run(Timer.java:505) 4) Thread[id=3407, name=qtp1114988144-3407, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=3397, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) at java.util.TimerThread.run(Timer.java:505) 6) Thread[id=3414, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221) at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=3416, name=Socket Reader #1 for port 38153, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) 8) Thread[id=3402, name=qtp1114988144-3402-acceptor-0@300e6f1d-ServerConnector@40de12fc{HTTP/1.1,[http/1.1]}{lucene2-us-west.apache.org:36343}, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=3405, name=qtp1114988144-3405, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=3401, name=qtp1114988144-3401, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/12111668.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=3403, name=qtp1114988144-3403, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=3400, name=qtp1114988144-3400, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/12111668.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=3404, name=qtp1114988144-3404, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=3408, name=Session-HouseKeeper-5161250c, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=3398, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@24b30f80, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest:
1) Thread[id=3399, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=3406, name=qtp1114988144-3406, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=3417, name=IPC Server idle connection scanner for port 38153, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
at java.util.TimerThread.run(Timer.java:505)
4) Thread[id=3407, name=qtp1114988144-3407, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=3397, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
at java.util.TimerThread.run(Timer.java:505)
6) Thread[id=3414, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)
at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=3416, name=Socket Reader #1 for port 38153, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
8) Thread[id=3402, name=qtp1114988144-3402-acceptor-0@300e6f1d-ServerConnector@40de12fc{HTTP/1.1,[http/1.1]}{lucene2-us-west.apache.org:36343}, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=3405, name=qtp1114988144-3405, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=3401, name=qtp1114988144-3401, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/12111668.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=3403, name=qtp1114988144-3403, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=3400, name=qtp1114988144-3400, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/12111668.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=3404, name=qtp1114988144-3404, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=3408, name=Session-HouseKeeper-5161250c, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=3398, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@24b30f80, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([D1C21E02EF7F767C]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=3399, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=3406, name=qtp1114988144-3406, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=3417, name=IPC Server idle connection scanner for port 38153, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) at java.util.TimerThread.run(Timer.java:505) 4) Thread[id=3407, name=qtp1114988144-3407, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=3397, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) at java.util.TimerThread.run(Timer.java:505) 6) Thread[id=3416, name=Socket Reader #1 for port 38153, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) 7) Thread[id=3402, name=qtp1114988144-3402, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=3405, name=qtp1114988144-3405, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=3401, name=qtp1114988144-3401, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/12111668.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=3403, name=qtp1114988144-3403, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=3400, name=qtp1114988144-3400, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/12111668.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=3404, name=qtp1114988144-3404, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=3408, name=Session-HouseKeeper-5161250c, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=3399, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=3406, name=qtp1114988144-3406, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=3417, name=IPC Server idle connection scanner for port 38153, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
at java.util.TimerThread.run(Timer.java:505)
4) Thread[id=3407, name=qtp1114988144-3407, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=3397, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
at java.util.TimerThread.run(Timer.java:505)
6) Thread[id=3416, name=Socket Reader #1 for port 38153, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.IOUtil.drain(Native Method)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
7) Thread[id=3402, name=qtp1114988144-3402, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=3405, name=qtp1114988144-3405, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=3401, name=qtp1114988144-3401, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/12111668.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=3403, name=qtp1114988144-3403, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=3400, name=qtp1114988144-3400, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/12111668.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=3404, name=qtp1114988144-3404, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=3408, name=Session-HouseKeeper-5161250c, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([D1C21E02EF7F767C]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestHdfsBackupRestoreCore
Error Message:
access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
at __randomizedtesting.SeedInfo.seed([D1C21E02EF7F767C]:0)
at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
at java.security.AccessController.checkPermission(AccessController.java:884)
at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
at org.apache.hadoop.util.Shell.run(Shell.java:901)
at org.apache.hadoop.fs.DF.getFilesystem(DF.java:74)
at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker$CheckedVolume.<init>(NameNodeResourceChecker.java:70)
at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.addDirToCheck(NameNodeResourceChecker.java:166)
at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.<init>(NameNodeResourceChecker.java:135)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startCommonServices(FSNamesystem.java:1168)
at org.apache.hadoop.hdfs.server.namenode.NameNode.startCommonServices(NameNode.java:791)
at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:717)
at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:940)
at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:913)
at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1646)
at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNode(MiniDFSCluster.java:1314)
at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1083)
at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
at org.apache.solr.handler.TestHdfsBackupRestoreCore.setupClass(TestHdfsBackupRestoreCore.java:106)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestHdfsBackupRestoreCore
Error Message:
15 threads leaked from SUITE scope at org.apache.solr.handler.TestHdfsBackupRestoreCore: 1) Thread[id=67, name=IPC Server idle connection scanner for port 39612, state=WAITING, group=TGRP-TestHdfsBackupRestoreCore] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) at java.util.TimerThread.run(Timer.java:505) 2) Thread[id=52, name=qtp1869227482-52, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/155039732.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=49, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=58, name=Session-HouseKeeper-6b72f723, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=50, name=qtp1869227482-50, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/155039732.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=51, name=qtp1869227482-51, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=48, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@275cc6db, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=66, name=Socket Reader #1 for port 39612, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) 9) Thread[id=53, name=qtp1869227482-53-acceptor-0@28dabb44-ServerConnector@1304731f{HTTP/1.1,[http/1.1]}{lucene2-us-west.apache.org:45735}, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=57, name=qtp1869227482-57, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=55, name=qtp1869227482-55, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=47, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) at java.util.TimerThread.run(Timer.java:505) 13) Thread[id=56, name=qtp1869227482-56, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=64, name=FSEditLogAsync, state=WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221) at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=54, name=qtp1869227482-54, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.handler.TestHdfsBackupRestoreCore:
1) Thread[id=67, name=IPC Server idle connection scanner for port 39612, state=WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
at java.util.TimerThread.run(Timer.java:505)
2) Thread[id=52, name=qtp1869227482-52, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/155039732.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=49, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=58, name=Session-HouseKeeper-6b72f723, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=50, name=qtp1869227482-50, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/155039732.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=51, name=qtp1869227482-51, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=48, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@275cc6db, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at java.lang.Thread.sleep(Native Method)
at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=66, name=Socket Reader #1 for port 39612, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
9) Thread[id=53, name=qtp1869227482-53-acceptor-0@28dabb44-ServerConnector@1304731f{HTTP/1.1,[http/1.1]}{lucene2-us-west.apache.org:45735}, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=57, name=qtp1869227482-57, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=55, name=qtp1869227482-55, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=47, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
at java.util.TimerThread.run(Timer.java:505)
13) Thread[id=56, name=qtp1869227482-56, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=64, name=FSEditLogAsync, state=WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)
at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=54, name=qtp1869227482-54, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([D1C21E02EF7F767C]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestHdfsBackupRestoreCore
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=67, name=IPC Server idle connection scanner for port 39612, state=WAITING, group=TGRP-TestHdfsBackupRestoreCore] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) at java.util.TimerThread.run(Timer.java:505) 2) Thread[id=52, name=qtp1869227482-52, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/155039732.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=49, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=58, name=Session-HouseKeeper-6b72f723, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=50, name=qtp1869227482-50, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/155039732.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=51, name=qtp1869227482-51, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=66, name=Socket Reader #1 for port 39612, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) 8) Thread[id=53, name=qtp1869227482-53, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=57, name=qtp1869227482-57, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=55, name=qtp1869227482-55, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=47, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) at java.util.TimerThread.run(Timer.java:505) 12) Thread[id=56, name=qtp1869227482-56, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=54, name=qtp1869227482-54, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=67, name=IPC Server idle connection scanner for port 39612, state=WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
at java.util.TimerThread.run(Timer.java:505)
2) Thread[id=52, name=qtp1869227482-52, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/155039732.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=49, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueue
[...truncated too long message...]
jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=2683, name=qtp108829400-2683, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=2676, name=qtp108829400-2676, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1497764942.run(Unknown Source)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 7) Thread[id=2684, name=Session-HouseKeeper-5b21ba47, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 8) Thread[id=2678, name=qtp108829400-2678, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 9) Thread[id=2692, name=Socket Reader #1 for port 42757, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
[junit4] > at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
[junit4] > at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
[junit4] > at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
[junit4] > at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
[junit4] > 10) Thread[id=2675, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 11) Thread[id=2680, name=qtp108829400-2680, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 12) Thread[id=2677, name=qtp108829400-2677, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1497764942.run(Unknown Source)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 13) Thread[id=2693, name=IPC Server idle connection scanner for port 42757, state=WAITING, group=TGRP-HdfsDirectoryTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at java.util.TimerThread.mainLoop(Timer.java:526)
[junit4] > at java.util.TimerThread.run(Timer.java:505)
[junit4] > at __randomizedtesting.SeedInfo.seed([D1C21E02EF7F767C]:0)
[junit4] Completed [746/887 (3!)] on J1 in 15.63s, 0 tests, 3 errors <<< FAILURES!
[...truncated 53349 lines...]