You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2017/05/14 02:03:14 UTC
[JENKINS] Lucene-Solr-NightlyTests-6.x - Build # 348 - Unstable
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.x/348/
2 tests failed.
FAILED: org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest.test
Error Message:
Error from server at http://127.0.0.1:45976/oneInstanceCollection22: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /oneInstanceCollection22/update. Reason: <pre> {metadata={error-class=org.apache.solr.core.SolrCoreInitializationException,root-error-class=java.lang.OutOfMemoryError},msg=SolrCore 'oneInstanceCollection22' is not available due to init failure: The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.,trace=org.apache.solr.core.SolrCoreInitializationException: SolrCore 'oneInstanceCollection22' is not available due to init failure: The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB. at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1313) at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:268) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:483) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) at org.eclipse.jetty.server.Server.handle(Server.java:534) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95) at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.solr.common.SolrException: The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB. at org.apache.solr.core.SolrCore.<init>(SolrCore.java:977) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:830) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:920) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:855) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:388) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:173) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:748) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:729) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:510) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462) ... 13 more Caused by: java.lang.RuntimeException: The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB. at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:312) at org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:288) at org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:228) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:346) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:91) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:728) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:923) ... 37 more Caused by: java.lang.OutOfMemoryError: Direct buffer memory at java.nio.Bits.reserveMemory(Bits.java:693) at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123) at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311) at org.apache.solr.store.blockcache.BlockCache.<init>(BlockCache.java:75) at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:310) ... 43 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/> </body> </html>
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:45976/oneInstanceCollection22: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /oneInstanceCollection22/update. Reason:
<pre> {metadata={error-class=org.apache.solr.core.SolrCoreInitializationException,root-error-class=java.lang.OutOfMemoryError},msg=SolrCore 'oneInstanceCollection22' is not available due to init failure: The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.,trace=org.apache.solr.core.SolrCoreInitializationException: SolrCore 'oneInstanceCollection22' is not available due to init failure: The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.
at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1313)
at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:268)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:483)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:534)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:977)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:830)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:920)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:855)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:388)
at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:173)
at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:748)
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:729)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:510)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
... 13 more
Caused by: java.lang.RuntimeException: The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.
at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:312)
at org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:288)
at org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:228)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:346)
at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:91)
at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:728)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:923)
... 37 more
Caused by: java.lang.OutOfMemoryError: Direct buffer memory
at java.nio.Bits.reserveMemory(Bits.java:693)
at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)
at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)
at org.apache.solr.store.blockcache.BlockCache.<init>(BlockCache.java:75)
at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:310)
... 43 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.14.v20161028</a><hr/>
</body>
</html>
at __randomizedtesting.SeedInfo.seed([1CFE855CF91E93CE:94AABA8657E2FE36]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:578)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:279)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:160)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
at org.apache.solr.cloud.BasicDistributedZkTest.testANewCollectionInOneInstanceWithManualShardAssignement(BasicDistributedZkTest.java:799)
at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:361)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.StressHdfsTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [NRTCachingDirectory] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347) at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:353) at org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3044) at org.apache.solr.core.SolrCore.close(SolrCore.java:1575) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:969) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:830) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:920) at org.apache.solr.core.CoreContainer.lambda$load$5(CoreContainer.java:558) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [NRTCachingDirectory]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:353)
at org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3044)
at org.apache.solr.core.SolrCore.close(SolrCore.java:1575)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:969)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:830)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:920)
at org.apache.solr.core.CoreContainer.lambda$load$5(CoreContainer.java:558)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([1CFE855CF91E93CE]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:302)
at sun.reflect.GeneratedMethodAccessor116.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 13306 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/init-core-data-001
[junit4] 2> 3452351 WARN (SUITE-HdfsBasicDistributedZkTest-seed#[1CFE855CF91E93CE]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=189 numCloses=189
[junit4] 2> 3452351 INFO (SUITE-HdfsBasicDistributedZkTest-seed#[1CFE855CF91E93CE]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields
[junit4] 2> 3452352 INFO (SUITE-HdfsBasicDistributedZkTest-seed#[1CFE855CF91E93CE]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 3452353 INFO (SUITE-HdfsBasicDistributedZkTest-seed#[1CFE855CF91E93CE]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 3452396 WARN (SUITE-HdfsBasicDistributedZkTest-seed#[1CFE855CF91E93CE]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 3452405 WARN (SUITE-HdfsBasicDistributedZkTest-seed#[1CFE855CF91E93CE]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3452407 INFO (SUITE-HdfsBasicDistributedZkTest-seed#[1CFE855CF91E93CE]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3452420 INFO (SUITE-HdfsBasicDistributedZkTest-seed#[1CFE855CF91E93CE]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_47987_hdfs____.h0weyp/webapp
[junit4] 2> 3452859 INFO (SUITE-HdfsBasicDistributedZkTest-seed#[1CFE855CF91E93CE]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:47987
[junit4] 2> 3452919 WARN (SUITE-HdfsBasicDistributedZkTest-seed#[1CFE855CF91E93CE]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3452920 INFO (SUITE-HdfsBasicDistributedZkTest-seed#[1CFE855CF91E93CE]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3452933 INFO (SUITE-HdfsBasicDistributedZkTest-seed#[1CFE855CF91E93CE]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_49292_datanode____.qwpp83/webapp
[junit4] 2> 3453346 INFO (SUITE-HdfsBasicDistributedZkTest-seed#[1CFE855CF91E93CE]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:49292
[junit4] 2> 3453437 INFO (IPC Server handler 5 on 51012) [ ] BlockStateChange BLOCK* processReport: from storage DS-76e2b8a0-249e-4d0e-95be-cdf4d6559ed8 node DatanodeRegistration(127.0.0.1:40108, datanodeUuid=9142dfcc-808e-4bba-9ff3-d1742d38e560, infoPort=35727, infoSecurePort=0, ipcPort=56598, storageInfo=lv=-56;cid=testClusterID;nsid=332336043;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs
[junit4] 2> 3453437 INFO (IPC Server handler 5 on 51012) [ ] BlockStateChange BLOCK* processReport: from storage DS-041d9309-bbbb-468e-86d0-6fce511669ef node DatanodeRegistration(127.0.0.1:40108, datanodeUuid=9142dfcc-808e-4bba-9ff3-d1742d38e560, infoPort=35727, infoSecurePort=0, ipcPort=56598, storageInfo=lv=-56;cid=testClusterID;nsid=332336043;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 3453505 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3453505 INFO (Thread-67038) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3453505 INFO (Thread-67038) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3453507 ERROR (Thread-67038) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3453605 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.ZkTestServer start zk server on port:58428
[junit4] 2> 3453619 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 3453621 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 3453622 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 3453624 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 3453625 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 3453626 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 3453628 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 3453629 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 3453630 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 3453631 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 3453633 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 3453752 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/control-001/cores/collection1
[junit4] 2> 3453754 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3453755 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@49ace355{/,null,AVAILABLE}
[junit4] 2> 3453756 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7c82baa{HTTP/1.1,[http/1.1]}{127.0.0.1:59041}
[junit4] 2> 3453757 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.e.j.s.Server Started @3456924ms
[junit4] 2> 3453757 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:51012/hdfs__localhost_51012__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-6.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001_tempDir-002_control_data, hostContext=/, hostPort=59041, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/control-001/cores}
[junit4] 2> 3453757 ERROR (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3453757 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.7.0
[junit4] 2> 3453757 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3453757 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3453757 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-14T01:29:40.082Z
[junit4] 2> 3453761 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3453761 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/control-001/solr.xml
[junit4] 2> 3453768 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 3453769 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58428/solr
[junit4] 2> 3453789 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [n:127.0.0.1:59041_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3453790 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [n:127.0.0.1:59041_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59041_
[junit4] 2> 3453791 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [n:127.0.0.1:59041_ ] o.a.s.c.Overseer Overseer (id=97958322492604420-127.0.0.1:59041_-n_0000000000) starting
[junit4] 2> 3453799 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [n:127.0.0.1:59041_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59041_
[junit4] 2> 3453800 INFO (zkCallback-3434-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3453947 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [n:127.0.0.1:59041_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/control-001/cores
[junit4] 2> 3453947 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [n:127.0.0.1:59041_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 3453948 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3453951 INFO (OverseerStateUpdate-97958322492604420-127.0.0.1:59041_-n_0000000000) [n:127.0.0.1:59041_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 3453974 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 3453990 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 3454116 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3454130 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection, trusted=true
[junit4] 2> 3454131 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:51012/solr_hdfs_home
[junit4] 2> 3454131 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 3454131 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3454131 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/control-001/cores/collection1], dataDir=[hdfs://localhost:51012/solr_hdfs_home/control_collection/core_node1/data/]
[junit4] 2> 3454131 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@41eeb539
[junit4] 2> 3454132 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51012/solr_hdfs_home/control_collection/core_node1/data/snapshot_metadata
[junit4] 2> 3454140 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3454140 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 3454140 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3454150 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3454150 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51012/solr_hdfs_home/control_collection/core_node1/data
[junit4] 2> 3454167 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51012/solr_hdfs_home/control_collection/core_node1/data/index
[junit4] 2> 3454174 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3454174 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 3454174 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3454188 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3454189 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=41, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 3454199 INFO (IPC Server handler 2 on 51012) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:40108 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-76e2b8a0-249e-4d0e-95be-cdf4d6559ed8:NORMAL:127.0.0.1:40108|RBW]]} size 0
[junit4] 2> 3454210 WARN (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3454264 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 3454264 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3454264 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 3454275 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3454276 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3454277 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=29.2421875, floorSegmentMB=0.8037109375, forceMergeDeletesPctAllowed=0.25638855136426497, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3780229629664714
[junit4] 2> 3454281 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@373a9505[collection1] main]
[junit4] 2> 3454282 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3454283 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3454283 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 3454287 INFO (searcherExecutor-6827-thread-1-processing-n:127.0.0.1:59041_ x:collection1 c:control_collection) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@373a9505[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3454291 WARN (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:51669/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:51669 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3454291 WARN (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:51669/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:51669 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3454292 INFO (coreLoadExecutor-6826-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1567333160705851392
[junit4] 2> 3454298 INFO (coreZkRegister-6819-thread-1-processing-n:127.0.0.1:59041_ x:collection1 c:control_collection) [n:127.0.0.1:59041_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3454298 INFO (coreZkRegister-6819-thread-1-processing-n:127.0.0.1:59041_ x:collection1 c:control_collection) [n:127.0.0.1:59041_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3454298 INFO (coreZkRegister-6819-thread-1-processing-n:127.0.0.1:59041_ x:collection1 c:control_collection) [n:127.0.0.1:59041_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:59041/collection1/
[junit4] 2> 3454298 INFO (coreZkRegister-6819-thread-1-processing-n:127.0.0.1:59041_ x:collection1 c:control_collection) [n:127.0.0.1:59041_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3454298 INFO (coreZkRegister-6819-thread-1-processing-n:127.0.0.1:59041_ x:collection1 c:control_collection) [n:127.0.0.1:59041_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:59041/collection1/ has no replicas
[junit4] 2> 3454298 INFO (coreZkRegister-6819-thread-1-processing-n:127.0.0.1:59041_ x:collection1 c:control_collection) [n:127.0.0.1:59041_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3454302 INFO (coreZkRegister-6819-thread-1-processing-n:127.0.0.1:59041_ x:collection1 c:control_collection) [n:127.0.0.1:59041_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:59041/collection1/ shard1
[junit4] 2> 3454453 INFO (coreZkRegister-6819-thread-1-processing-n:127.0.0.1:59041_ x:collection1 c:control_collection) [n:127.0.0.1:59041_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3454458 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3454459 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:58428/solr ready
[junit4] 2> 3454459 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 3454459 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
[junit4] 2> 3454577 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/shard-1-001/cores/collection1
[junit4] 2> 3454577 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/shard-1-001
[junit4] 2> 3454578 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3454579 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@419dae98{/,null,AVAILABLE}
[junit4] 2> 3454580 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@50bcb3af{HTTP/1.1,[http/1.1]}{127.0.0.1:45976}
[junit4] 2> 3454581 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.e.j.s.Server Started @3457748ms
[junit4] 2> 3454581 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:51012/hdfs__localhost_51012__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-6.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001_tempDir-002_jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=45976, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/shard-1-001/cores}
[junit4] 2> 3454581 ERROR (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3454581 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.7.0
[junit4] 2> 3454581 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3454581 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3454581 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-14T01:29:40.906Z
[junit4] 2> 3454584 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3454584 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/shard-1-001/solr.xml
[junit4] 2> 3454590 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 3454591 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58428/solr
[junit4] 2> 3454600 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [n:127.0.0.1:45976_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3454603 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [n:127.0.0.1:45976_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3454605 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [n:127.0.0.1:45976_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45976_
[junit4] 2> 3454607 INFO (zkCallback-3434-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3454607 INFO (zkCallback-3438-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3454610 INFO (zkCallback-3444-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3454657 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [n:127.0.0.1:45976_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/shard-1-001/cores
[junit4] 2> 3454657 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [n:127.0.0.1:45976_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 3454659 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3454660 INFO (OverseerStateUpdate-97958322492604420-127.0.0.1:59041_-n_0000000000) [n:127.0.0.1:59041_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
[junit4] 2> 3454762 INFO (zkCallback-3444-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3455674 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 3455690 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 3455814 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3455826 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1, trusted=true
[junit4] 2> 3455826 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:51012/solr_hdfs_home
[junit4] 2> 3455826 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 3455826 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3455826 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/shard-1-001/cores/collection1], dataDir=[hdfs://localhost:51012/solr_hdfs_home/collection1/core_node1/data/]
[junit4] 2> 3455827 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@41eeb539
[junit4] 2> 3455827 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51012/solr_hdfs_home/collection1/core_node1/data/snapshot_metadata
[junit4] 2> 3455835 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3455835 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 3455835 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3455850 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3455851 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51012/solr_hdfs_home/collection1/core_node1/data
[junit4] 2> 3455869 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51012/solr_hdfs_home/collection1/core_node1/data/index
[junit4] 2> 3455876 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3455876 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 3455876 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3455896 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3455896 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=41, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 3455913 INFO (IPC Server handler 7 on 51012) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:40108 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-041d9309-bbbb-468e-86d0-6fce511669ef:NORMAL:127.0.0.1:40108|RBW]]} size 0
[junit4] 2> 3455921 WARN (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3455968 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 3455968 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3455968 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 3455984 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3455984 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3455986 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=29.2421875, floorSegmentMB=0.8037109375, forceMergeDeletesPctAllowed=0.25638855136426497, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3780229629664714
[junit4] 2> 3455990 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@43c5046a[collection1] main]
[junit4] 2> 3455991 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3455991 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3455991 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 3455993 INFO (searcherExecutor-6838-thread-1-processing-n:127.0.0.1:45976_ x:collection1 c:collection1) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@43c5046a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3456000 WARN (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:51669/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:51669 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3456001 WARN (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:51669/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:51669 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3456001 WARN (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:51669/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:51669 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3456002 INFO (coreLoadExecutor-6837-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1567333162498916352
[junit4] 2> 3456008 INFO (coreZkRegister-6832-thread-1-processing-n:127.0.0.1:45976_ x:collection1 c:collection1) [n:127.0.0.1:45976_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3456009 INFO (coreZkRegister-6832-thread-1-processing-n:127.0.0.1:45976_ x:collection1 c:collection1) [n:127.0.0.1:45976_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3456009 INFO (coreZkRegister-6832-thread-1-processing-n:127.0.0.1:45976_ x:collection1 c:collection1) [n:127.0.0.1:45976_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45976/collection1/
[junit4] 2> 3456009 INFO (coreZkRegister-6832-thread-1-processing-n:127.0.0.1:45976_ x:collection1 c:collection1) [n:127.0.0.1:45976_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3456009 INFO (coreZkRegister-6832-thread-1-processing-n:127.0.0.1:45976_ x:collection1 c:collection1) [n:127.0.0.1:45976_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:45976/collection1/ has no replicas
[junit4] 2> 3456010 INFO (coreZkRegister-6832-thread-1-processing-n:127.0.0.1:45976_ x:collection1 c:collection1) [n:127.0.0.1:45976_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3456013 INFO (coreZkRegister-6832-thread-1-processing-n:127.0.0.1:45976_ x:collection1 c:collection1) [n:127.0.0.1:45976_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45976/collection1/ shard2
[junit4] 2> 3456115 INFO (zkCallback-3444-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3456165 INFO (coreZkRegister-6832-thread-1-processing-n:127.0.0.1:45976_ x:collection1 c:collection1) [n:127.0.0.1:45976_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3456269 INFO (zkCallback-3444-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3456277 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/shard-2-001/cores/collection1
[junit4] 2> 3456277 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/shard-2-001
[junit4] 2> 3456278 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3456279 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4d5b3ee1{/,null,AVAILABLE}
[junit4] 2> 3456286 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@57c85cbd{HTTP/1.1,[http/1.1]}{127.0.0.1:48338}
[junit4] 2> 3456287 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.e.j.s.Server Started @3459454ms
[junit4] 2> 3456287 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:51012/hdfs__localhost_51012__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-6.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001_tempDir-002_jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=48338, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/shard-2-001/cores}
[junit4] 2> 3456287 ERROR (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3456287 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.7.0
[junit4] 2> 3456287 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3456287 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3456287 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-14T01:29:42.612Z
[junit4] 2> 3456290 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3456290 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/shard-2-001/solr.xml
[junit4] 2> 3456309 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 3456309 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58428/solr
[junit4] 2> 3456318 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [n:127.0.0.1:48338_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3456321 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [n:127.0.0.1:48338_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3456323 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [n:127.0.0.1:48338_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48338_
[junit4] 2> 3456325 INFO (zkCallback-3444-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3456325 INFO (zkCallback-3434-thread-1-processing-n:127.0.0.1:59041_) [n:127.0.0.1:59041_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3456325 INFO (zkCallback-3438-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3456325 INFO (zkCallback-3450-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3456386 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [n:127.0.0.1:48338_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/shard-2-001/cores
[junit4] 2> 3456386 INFO (TEST-HdfsBasicDistributedZkTest.test-seed#[1CFE855CF91E93CE]) [n:127.0.0.1:48338_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 3456390 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3456391 INFO (OverseerStateUpdate-97958322492604420-127.0.0.1:59041_-n_0000000000) [n:127.0.0.1:59041_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 3456493 INFO (zkCallback-3444-thread-1-processing-n:127.0.0.1:45976_) [n:127.0.0.1:45976_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 3456493 INFO (zkCallback-3450-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 3457405 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 3457423 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 3457553 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3457567 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1, trusted=true
[junit4] 2> 3457567 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:51012/solr_hdfs_home
[junit4] 2> 3457567 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 3457567 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3457567 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZkTest_1CFE855CF91E93CE-001/shard-2-001/cores/collection1], dataDir=[hdfs://localhost:51012/solr_hdfs_home/collection1/core_node2/data/]
[junit4] 2> 3457568 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@41eeb539
[junit4] 2> 3457568 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51012/solr_hdfs_home/collection1/core_node2/data/snapshot_metadata
[junit4] 2> 3457577 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3457578 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 3457578 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3457590 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3457590 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51012/solr_hdfs_home/collection1/core_node2/data
[junit4] 2> 3457610 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51012/solr_hdfs_home/collection1/core_node2/data/index
[junit4] 2> 3457619 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3457620 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 3457620 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3457631 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3457631 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=41, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 3457640 INFO (IPC Server handler 6 on 51012) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:40108 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-76e2b8a0-249e-4d0e-95be-cdf4d6559ed8:NORMAL:127.0.0.1:40108|FINALIZED]]} size 0
[junit4] 2> 3457649 WARN (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3457699 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 3457699 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3457699 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 3457710 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3457711 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3457713 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=29.2421875, floorSegmentMB=0.8037109375, forceMergeDeletesPctAllowed=0.25638855136426497, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3780229629664714
[junit4] 2> 3457716 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@76e97d2[collection1] main]
[junit4] 2> 3457718 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3457718 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3457718 INFO (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 3457722 INFO (searcherExecutor-6849-thread-1-processing-n:127.0.0.1:48338_ x:collection1 c:collection1) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@76e97d2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3457725 WARN (coreLoadExecutor-6848-thread-1-processing-n:127.0.0.1:48338_) [n:127.0.0.1:48338_ c:collection1 x:collection1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:51669/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:51669 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3457725 WARN (coreLoadExecutor-6848-thre
[...truncated too long message...]
readPoolExecutor.java:1066)
[junit4] 2> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] 2> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> IPC Parameter Sending Thread #6:
[junit4] 2> [...sun.*]
[junit4] 2> java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] 2> java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] 2> java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] 2> java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] 2> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] 2> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] 2> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> JUnit4-serializer-daemon:
[junit4] 2> java.lang.Thread.sleep(Native Method)
[junit4] 2> com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
[junit4] 2>
[junit4] 2> Finalizer:
[junit4] 2> java.lang.Object.wait(Native Method)
[junit4] 2> java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
[junit4] 2> java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
[junit4] 2> java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)
[junit4] 2>
[junit4] 2> 3822533 INFO (SUITE-StressHdfsTest-seed#[1CFE855CF91E93CE]-worker) [ ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {multiDefault=PostingsFormat(name=Direct), id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), text=PostingsFormat(name=LuceneVarGapDocFreqInterval), txt_t=PostingsFormat(name=LuceneVarGapDocFreqInterval)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Lucene54), _version_=DocValuesFormat(name=Direct), multiDefault=DocValuesFormat(name=Lucene54), intDefault=DocValuesFormat(name=Direct), range_facet_i_dv=DocValuesFormat(name=Direct), id=DocValuesFormat(name=Lucene54), text=DocValuesFormat(name=Asserting), intDvoDefault=DocValuesFormat(name=Asserting), range_facet_l=DocValuesFormat(name=Direct), timestamp=DocValuesFormat(name=Direct), txt_t=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=27, maxMBSortInHeap=6.9544080595409685, sim=RandomSimilarity(queryNorm=false,coord=no): {}, locale=es-PE, timezone=America/Anchorage
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=4,threads=2,free=253585088,total=528482304
[junit4] 2> NOTE: All tests run in this JVM: [TestBinaryResponseWriter, TestReplicaProperties, TestComponentsName, RankQueryTest, CollectionsAPIAsyncDistributedZkTest, ShufflingReplicaListTransformerTest, TestCloudRecovery, MBeansHandlerTest, TestDocSet, TestDownShardTolerantSearch, DistanceUnitsTest, HLLSerializationTest, TlogReplayBufferedWhileIndexingTest, RegexBoostProcessorTest, TestApiFramework, TestAnalyzeInfixSuggestions, TestMaxScoreQueryParser, HdfsChaosMonkeySafeLeaderTest, DistributedFacetPivotSmallAdvancedTest, TestSchemaSimilarityResource, TestFaceting, QueryEqualityTest, TestLRUStatsCache, TestSolrFieldCacheMBean, NotRequiredUniqueKeyTest, TestSQLHandler, TestRealTimeGet, XsltUpdateRequestHandlerTest, ReturnFieldsTest, TestDocumentBuilder, TestCopyFieldCollectionResource, TestRangeQuery, PreAnalyzedFieldManagedSchemaCloudTest, TestPHPSerializedResponseWriter, TestFreeTextSuggestions, TestFieldTypeResource, OutputWriterTest, TestReplicationHandler, TestStressLiveNodes, ClassificationUpdateProcessorIntegrationTest, TestBlobHandler, SuggestComponentTest, FileBasedSpellCheckerTest, SyncSliceTest, TestRandomFlRTGCloud, DistributedQueryComponentCustomSortTest, BlockDirectoryTest, TestManagedSchemaAPI, SolrCLIZkUtilsTest, CoreAdminCreateDiscoverTest, RequiredFieldsTest, SecurityConfHandlerTest, TestValueSourceCache, TestReplicationHandlerBackup, ResponseHeaderTest, HighlighterConfigTest, OverseerCollectionConfigSetProcessorTest, PolyFieldTest, TestCollationFieldDocValues, TestStressRecovery, TestRTimerTree, DistributedFacetExistsSmallTest, TestSolrConfigHandler, TestPseudoReturnFields, UniqFieldsUpdateProcessorFactoryTest, DocValuesMissingTest, AliasIntegrationTest, TestHighlightDedupGrouping, DisMaxRequestHandlerTest, TestSolr4Spatial, ExitableDirectoryReaderTest, ResourceLoaderTest, TestLazyCores, BaseCdcrDistributedZkTest, TestStressLucene, QueryElevationComponentTest, CustomCollectionTest, TestAuthenticationFramework, TestBackupRepositoryFactory, TestFieldCollectionResource, TestClusterStateMutator, TestFastLRUCache, DefaultValueUpdateProcessorTest, TestCloudManagedSchema, BasicDistributedZkTest, TestCustomDocTransformer, CdcrReplicationHandlerTest, TestSSLRandomization, TestOmitPositions, TestFastWriter, TestSolrCloudSnapshots, CdcrUpdateLogTest, TestCustomStream, BooleanFieldTest, FastVectorHighlighterTest, BitVectorTest, TestCSVLoader, TestSizeLimitedDistributedMap, TestNamedUpdateProcessors, ChaosMonkeyNothingIsSafeTest, TestSolrDeletionPolicy2, LeaderElectionIntegrationTest, SolrGraphiteReporterTest, TestAtomicUpdateErrorCases, JvmMetricsTest, ConfigureRecoveryStrategyTest, TestImpersonationWithHadoopAuth, TestReloadAndDeleteDocs, SynonymTokenizerTest, DeleteNodeTest, DistributedSpellCheckComponentTest, RecoveryAfterSoftCommitTest, TestPerFieldSimilarity, TestUseDocValuesAsStored2, FacetPivotSmallTest, CacheHeaderTest, FieldMutatingUpdateProcessorTest, DirectUpdateHandlerOptimizeTest, DistanceFunctionTest, IndexBasedSpellCheckerTest, LukeRequestHandlerTest, DirectSolrSpellCheckerTest, TestQueryTypes, PrimitiveFieldTypeTest, XmlUpdateRequestHandlerTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, FieldAnalysisRequestHandlerTest, IndexSchemaRuntimeFieldTest, TestJmxIntegration, SearchHandlerTest, AlternateDirectoryTest, TestSolrIndexConfig, TestQuerySenderNoQuery, CopyFieldTest, BadComponentTest, MultiTermTest, SampleTest, TestBinaryField, TestSearchPerf, MinimalSchemaTest, ExternalFileFieldSortTest, TestSolrCoreProperties, TestLuceneMatchVersion, TestXIncludeConfig, TestLMJelinekMercerSimilarityFactory, TestIBSimilarityFactory, TestBM25SimilarityFactory, ScriptEngineTest, TestSuggestSpellingConverter, DateFieldTest, CursorPagingTest, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestSimpleTrackingShardHandler, TestTolerantSearch, TestPathTrie, TestEmbeddedSolrServerSchemaAPI, TestJettySolrRunner, ConnectionReuseTest, AssignTest, CdcrBootstrapTest, ConcurrentDeleteAndCreateCollectionTest, DistribJoinFromCollectionTest, LeaderElectionContextKeyTest, LeaderFailoverAfterPartitionTest, MigrateRouteKeyTest, OverseerTaskQueueTest, ReplicationFactorTest, SSLMigrationTest, ShardSplitTest, TriLevelCompositeIdRoutingTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsCollectionsAPIDistributedZkTest, HdfsUnloadDistributedZkTest, StressHdfsTest]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=StressHdfsTest -Dtests.seed=1CFE855CF91E93CE -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-PE -Dtests.timezone=America/Anchorage -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 0.00s J0 | StressHdfsTest (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [NRTCachingDirectory]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
[junit4] > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
[junit4] > at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:353)
[junit4] > at org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3044)
[junit4] > at org.apache.solr.core.SolrCore.close(SolrCore.java:1575)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:969)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:830)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:920)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$5(CoreContainer.java:558)
[junit4] > at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([1CFE855CF91E93CE]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:302)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] Completed [563/711 (2!)] on J0 in 103.35s, 1 test, 1 failure <<< FAILURES!
[...truncated 43873 lines...]
[JENKINS] Lucene-Solr-NightlyTests-6.x - Build # 350 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.x/350/
2 tests failed.
FAILED: org.apache.lucene.index.TestIndexSorting.testRandom3
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
at __randomizedtesting.SeedInfo.seed([C9B802A09D538A0A:6B604C7AF9A1A30C]:0)
at org.apache.lucene.util.packed.Packed8ThreeBlocks.<init>(Packed8ThreeBlocks.java:41)
at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:963)
at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:939)
at org.apache.lucene.util.packed.GrowableWriter.ensureCapacity(GrowableWriter.java:80)
at org.apache.lucene.util.packed.GrowableWriter.set(GrowableWriter.java:88)
at org.apache.lucene.util.packed.AbstractPagedMutable.set(AbstractPagedMutable.java:98)
at org.apache.lucene.util.fst.NodeHash.addNew(NodeHash.java:152)
at org.apache.lucene.util.fst.NodeHash.rehash(NodeHash.java:169)
at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:133)
at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:214)
at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:310)
at org.apache.lucene.util.fst.Builder.add(Builder.java:414)
at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.writeFST(MemoryDocValuesConsumer.java:367)
at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:404)
at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedField(DocValuesConsumer.java:653)
at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:204)
at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:153)
at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:167)
at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:111)
at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4356)
at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3931)
at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2083)
at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:5005)
at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5043)
at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5034)
at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1574)
at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1316)
at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2229)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
FAILED: org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.testCollectionsAPI
Error Message:
expected:<3> but was:<1>
Stack Trace:
java.lang.AssertionError: expected:<3> but was:<1>
at __randomizedtesting.SeedInfo.seed([1F377C89C3176F6:4986037C9A025963]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:522)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 1819 lines...]
[junit4] Suite: org.apache.lucene.index.TestIndexSorting
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestIndexSorting -Dtests.method=testRandom3 -Dtests.seed=C9B802A09D538A0A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/test-data/enwiki.random.lines.txt -Dtests.locale=ca-ES -Dtests.timezone=PNT -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 3851s J1 | TestIndexSorting.testRandom3 <<<
[junit4] > Throwable #1: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([C9B802A09D538A0A:6B604C7AF9A1A30C]:0)
[junit4] > at org.apache.lucene.util.packed.Packed8ThreeBlocks.<init>(Packed8ThreeBlocks.java:41)
[junit4] > at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:963)
[junit4] > at org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:939)
[junit4] > at org.apache.lucene.util.packed.GrowableWriter.ensureCapacity(GrowableWriter.java:80)
[junit4] > at org.apache.lucene.util.packed.GrowableWriter.set(GrowableWriter.java:88)
[junit4] > at org.apache.lucene.util.packed.AbstractPagedMutable.set(AbstractPagedMutable.java:98)
[junit4] > at org.apache.lucene.util.fst.NodeHash.addNew(NodeHash.java:152)
[junit4] > at org.apache.lucene.util.fst.NodeHash.rehash(NodeHash.java:169)
[junit4] > at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:133)
[junit4] > at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:214)
[junit4] > at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:310)
[junit4] > at org.apache.lucene.util.fst.Builder.add(Builder.java:414)
[junit4] > at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.writeFST(MemoryDocValuesConsumer.java:367)
[junit4] > at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.addSortedField(MemoryDocValuesConsumer.java:404)
[junit4] > at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedField(DocValuesConsumer.java:653)
[junit4] > at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:204)
[junit4] > at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:153)
[junit4] > at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:167)
[junit4] > at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:111)
[junit4] > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4356)
[junit4] > at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3931)
[junit4] > at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
[junit4] > at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2083)
[junit4] > at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:5005)
[junit4] > at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
[junit4] > at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5043)
[junit4] > at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5034)
[junit4] > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1574)
[junit4] > at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1316)
[junit4] > at org.apache.lucene.index.TestIndexSorting.testRandom3(TestIndexSorting.java:2229)
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/build/core/test/J1/temp/lucene.index.TestIndexSorting_C9B802A09D538A0A-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {docs=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), sparse_text=PostingsFormat(name=LuceneVarGapDocFreqInterval), norms=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), positions=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), term_vectors=PostingsFormat(name=Memory doPackFST= true)}, docValues:{multi_valued_long=DocValuesFormat(name=Asserting), foo=DocValuesFormat(name=Asserting), numeric=DocValuesFormat(name=Direct), float=DocValuesFormat(name=Lucene54), multi_valued_numeric=DocValuesFormat(name=Memory), long=DocValuesFormat(name=Lucene54), points=DocValuesFormat(name=Memory), sparse_int=DocValuesFormat(name=Lucene54), bar=DocValuesFormat(name=Memory), multi_valued_double=DocValuesFormat(name=Memory), docs=DocValuesFormat(name=Memory), id=DocValuesFormat(name=Memory), multi_valued_int=DocValuesFormat(name=Direct), multi_valued_bytes=DocValuesFormat(name=Direct), sparse_text=DocValuesFormat(name=Asserting), double=DocValuesFormat(name=Direct), positions=DocValuesFormat(name=Asserting), sparse_binary=DocValuesFormat(name=Asserting), int=DocValuesFormat(name=Memory), sorted=DocValuesFormat(name=Direct), sparse=DocValuesFormat(name=Asserting), multi_valued_string=DocValuesFormat(name=Memory), norms=DocValuesFormat(name=Memory), bytes=DocValuesFormat(name=Memory), binary=DocValuesFormat(name=Direct), dense_int=DocValuesFormat(name=Memory), multi_valued_float=DocValuesFormat(name=Lucene54), term_vectors=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=25, maxMBSortInHeap=5.2194053838463725, sim=RandomSimilarity(queryNorm=true,coord=yes): {positions=DFR I(ne)L3(800.0), id=DFR GL3(800.0), term_vectors=DFR I(n)1}, locale=ca-ES, timezone=PNT
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=4,threads=1,free=335686232,total=422051840
[junit4] 2> NOTE: All tests run in this JVM: [TestTermsEnum, TestBooleanQueryVisitSubscorers, TestFilterIterator, TestConcurrentMergeScheduler, TestWildcard, TestMultiTermConstantScore, TestUsageTrackingFilterCachingPolicy, TestIndexWriterCommit, TestLucene60PointsFormat, TestExternalCodecs, TestLSBRadixSorter, TestManyFields, TestSubScorerFreqs, TestTimSorterWorstCase, TestFilterWeight, TestReaderClosed, TestVirtualMethod, TestAllFilesHaveChecksumFooter, TestArrayUtil, TestIsCurrent, TestSloppyPhraseQuery, TestAutomaton, TestQueryRescorer, TestRecyclingByteBlockAllocator, TestReaderWrapperDVTypeCheck, TestDirectoryReader, TestNRTReaderCleanup, Test2BSortedDocValuesOrds, TestForTooMuchCloning, TestCheckIndex, TestSimpleAttributeImpl, TestRadixSelector, TestSpanNearQuery, TestParallelLeafReader, TestOmitNorms, TestBinaryDocument, TestIndexWriterOnVMError, TestSentinelIntSet, TestFilterLeafReader, TestDocValuesRewriteMethod, TestTopDocsCollector, TestDocIDMerger, TestAutomatonQuery, TestStressNRT, TestRAMDirectory, Test2BBKDPoints, TestIntArrayDocIdSet, TestNGramPhraseQuery, TestPagedBytes, TestUpgradeIndexMergePolicy, TestNorms, TestIndexWriterOnDiskFull, TestMultiThreadTermVectors, TestLucene50StoredFieldsFormat, TestNoMergePolicy, TestRegexpRandom, TestPostingsOffsets, TestLongRangeFieldQueries, TestUnicodeUtil, TestPriorityQueue, TestMultiMMap, TestSloppyMath, TestDuelingCodecsAtNight, TestCharsRefBuilder, TestPerFieldPostingsFormat, TestSameScoresWithThreads, TestIntBlockPool, TestNIOFSDirectory, TestCodecUtil, TestSizeBoundedForceMerge, TestGraphTokenizers, TestIndexSearcher, TestDeterminism, TestOmitTf, TestMathUtil, TestNeverDelete, TestComplexExplanations, TestFastDecompressionMode, TestParallelCompositeReader, TestFieldCacheRewriteMethod, TestTransactions, TestSegmentTermDocs, TestPerFieldPostingsFormat2, TestSort, TestMultiDocValues, TestDocValuesIndexing, TestBooleanMinShouldMatch, TestLegacyNumericUtils, TestSpans, TestCustomNorms, TestTransactionRollback, TestSpanExplanations, TestMultiPhraseQuery, TestMergeSchedulerExternal, TestBagOfPostings, TestFuzzyQuery, TestDoc, TestSimilarity2, TestNoDeletionPolicy, TestTermVectorsReader, TestMultiValuedNumericRangeQuery, TestComplexExplanationsOfNonMatches, TestSortRandom, TestPrefixCodedTerms, TestAllFilesHaveCodecHeader, TestParallelReaderEmptyIndex, TestDocument, TestMaxTermFrequency, TestTermScorer, TestTopFieldCollector, TestFileSwitchDirectory, TestSumDocFreq, TestIndexableField, TestWildcardRandom, TestDirectory, TestMultiLevelSkipList, TestIndexFileDeleter, TestConstantScoreQuery, TestMatchAllDocsQuery, TestBinaryTerms, TestConjunctions, TestMultiTermQueryRewrites, TestPositionIncrement, TestField, TestDateSort, TestSimilarityProvider, TestNumericTokenStream, TestSearch, TestCachingTokenFilter, TestDocCount, TestCachingCollector, TestAttributeSource, TestTotalHitCountCollector, TestBooleanScorer, TestRamUsageEstimator, TestSpanFirstQuery, TestIndexWriterLockRelease, TestPhrasePrefixQuery, TestByteBlockPool, TestBytesRef, TestDemo, TestCharFilter, TestRollback, TestTwoPhaseCommitTool, TestNot, TestCloseableThreadLocal, TestIndexWriterOnJRECrash, TestTermdocPerf, Test2BPositions, TestIndexCommit, TestTerm, TestPositiveScoresOnlyCollector, Test2BTerms, TestNewestSegment, TestByteArrayDataInput, Test2BNumericDocValues, Test2BBinaryDocValues, TestCharArrayMap, TestCharArraySet, TestCharacterUtils, TestReusableStringReader, TestStopFilter, TestBytesRefAttImpl, TestPackedTokenAttributeImpl, TestBlockPostingsFormat3, TestLucene50CompoundFormat, TestLucene50StoredFieldsFormatHighCompression, TestLucene50TermVectorsFormat, TestLucene60FieldInfoFormat, TestLucene53NormsFormat, TestFieldType, TestGeoEncodingUtils, TestPolygon, TestPolygon2D, TestExitableDirectoryReader, TestFieldReuse, TestFilterCodecReader, TestIndexReaderClose, TestIndexSorting]
[junit4] Completed [456/456 (1!)] on J1 in 3879.00s, 50 tests, 1 error <<< FAILURES!
[...truncated 1 lines...]
[junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/build/core/test/temp/junit4-J1-20170516_015011_9675172344729003322175.sysout
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/heapdumps/java_pid1694.hprof ...
[junit4] Heap dump file created [391349138 bytes in 1.392 secs]
[junit4] <<< JVM J1: EOF ----
[...truncated 9968 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1F377C89C3176F6-001/init-core-data-001
[junit4] 2> 513 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields
[junit4] 2> 516 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 881 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1F377C89C3176F6-001/tempDir-001
[junit4] 2> 892 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 894 INFO (Thread-1) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 894 INFO (Thread-1) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 994 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:42404
[junit4] 2> 1069 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
[junit4] 2> 1075 ERROR (Thread-1) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3427 INFO (jetty-launcher-1-thread-4) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3429 INFO (jetty-launcher-1-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3429 INFO (jetty-launcher-1-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3430 INFO (jetty-launcher-1-thread-3) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3490 INFO (jetty-launcher-1-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@26f6967a{/solr,null,AVAILABLE}
[junit4] 2> 3494 INFO (jetty-launcher-1-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@312f59a7{/solr,null,AVAILABLE}
[junit4] 2> 3490 INFO (jetty-launcher-1-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6fa2f9{/solr,null,AVAILABLE}
[junit4] 2> 3501 INFO (jetty-launcher-1-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@66f66c72{/solr,null,AVAILABLE}
[junit4] 2> 3547 INFO (jetty-launcher-1-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3daf36b9{SSL,[ssl, http/1.1]}{127.0.0.1:50939}
[junit4] 2> 3547 INFO (jetty-launcher-1-thread-1) [ ] o.e.j.s.Server Started @6650ms
[junit4] 2> 3547 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50939}
[junit4] 2> 3554 INFO (jetty-launcher-1-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7c347514{SSL,[ssl, http/1.1]}{127.0.0.1:56481}
[junit4] 2> 3565 INFO (jetty-launcher-1-thread-2) [ ] o.e.j.s.Server Started @6668ms
[junit4] 2> 3554 INFO (jetty-launcher-1-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7de2898e{SSL,[ssl, http/1.1]}{127.0.0.1:47037}
[junit4] 2> 3567 INFO (jetty-launcher-1-thread-4) [ ] o.e.j.s.Server Started @6670ms
[junit4] 2> 3567 INFO (jetty-launcher-1-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=47037}
[junit4] 2> 3554 INFO (jetty-launcher-1-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4fcf6af4{SSL,[ssl, http/1.1]}{127.0.0.1:38868}
[junit4] 2> 3565 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=56481}
[junit4] 2> 3574 INFO (jetty-launcher-1-thread-3) [ ] o.e.j.s.Server Started @6677ms
[junit4] 2> 3574 INFO (jetty-launcher-1-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38868}
[junit4] 2> 3635 ERROR (jetty-launcher-1-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3638 ERROR (jetty-launcher-1-thread-4) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3639 INFO (jetty-launcher-1-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.7.0
[junit4] 2> 3645 INFO (jetty-launcher-1-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3649 INFO (jetty-launcher-1-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3649 ERROR (jetty-launcher-1-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3650 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.7.0
[junit4] 2> 3650 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3650 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3645 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.7.0
[junit4] 2> 3651 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3651 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3739 ERROR (jetty-launcher-1-thread-3) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3739 INFO (jetty-launcher-1-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.7.0
[junit4] 2> 3739 INFO (jetty-launcher-1-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3739 INFO (jetty-launcher-1-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3764 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-16T06:10:19.784Z
[junit4] 2> 3766 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-16T06:10:19.784Z
[junit4] 2> 3766 INFO (jetty-launcher-1-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-16T06:10:19.782Z
[junit4] 2> 3766 INFO (jetty-launcher-1-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-16T06:10:19.784Z
[junit4] 2> 3817 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
[junit4] 2> 3847 INFO (jetty-launcher-1-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3871 INFO (jetty-launcher-1-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3883 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3883 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 4071 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15c0fe0e9330004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 4408 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 4410 INFO (jetty-launcher-1-thread-3) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 4412 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 4434 INFO (jetty-launcher-1-thread-4) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 4532 INFO (jetty-launcher-1-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42404/solr
[junit4] 2> 4534 INFO (jetty-launcher-1-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42404/solr
[junit4] 2> 4534 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42404/solr
[junit4] 2> 4546 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42404/solr
[junit4] 2> 4770 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15c0fe0e9330008, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 4800 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15c0fe0e9330006, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 5170 INFO (jetty-launcher-1-thread-4) [n:127.0.0.1:47037_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5170 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:50939_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5173 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:56481_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5176 INFO (jetty-launcher-1-thread-4) [n:127.0.0.1:47037_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:47037_solr
[junit4] 2> 5180 INFO (jetty-launcher-1-thread-3) [n:127.0.0.1:38868_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5182 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:50939_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50939_solr
[junit4] 2> 5182 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:56481_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56481_solr
[junit4] 2> 5185 INFO (jetty-launcher-1-thread-4) [n:127.0.0.1:47037_solr ] o.a.s.c.Overseer Overseer (id=97970750554832907-127.0.0.1:47037_solr-n_0000000000) starting
[junit4] 2> 5242 INFO (zkCallback-22-thread-1-processing-n:127.0.0.1:47037_solr) [n:127.0.0.1:47037_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 5244 INFO (zkCallback-23-thread-1-processing-n:127.0.0.1:50939_solr) [n:127.0.0.1:50939_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 5242 INFO (zkCallback-21-thread-1-processing-n:127.0.0.1:56481_solr) [n:127.0.0.1:56481_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 5244 INFO (zkCallback-20-thread-1-processing-n:127.0.0.1:38868_solr) [n:127.0.0.1:38868_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 5274 INFO (jetty-launcher-1-thread-3) [n:127.0.0.1:38868_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38868_solr
[junit4] 2> 5294 INFO (zkCallback-23-thread-1-processing-n:127.0.0.1:50939_solr) [n:127.0.0.1:50939_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5295 INFO (zkCallback-20-thread-1-processing-n:127.0.0.1:38868_solr) [n:127.0.0.1:38868_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5294 INFO (zkCallback-21-thread-1-processing-n:127.0.0.1:56481_solr) [n:127.0.0.1:56481_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5302 INFO (zkCallback-22-thread-1-processing-n:127.0.0.1:47037_solr) [n:127.0.0.1:47037_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5765 INFO (jetty-launcher-1-thread-4) [n:127.0.0.1:47037_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47037_solr
[junit4] 2> 5769 INFO (zkCallback-23-thread-1-processing-n:127.0.0.1:50939_solr) [n:127.0.0.1:50939_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5769 INFO (zkCallback-21-thread-1-processing-n:127.0.0.1:56481_solr) [n:127.0.0.1:56481_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5769 INFO (zkCallback-20-thread-1-processing-n:127.0.0.1:38868_solr) [n:127.0.0.1:38868_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5770 INFO (zkCallback-22-thread-1-processing-n:127.0.0.1:47037_solr) [n:127.0.0.1:47037_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 6009 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:56481_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1F377C89C3176F6-001/tempDir-001/node2/.
[junit4] 2> 6014 INFO (jetty-launcher-1-thread-3) [n:127.0.0.1:38868_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1F377C89C3176F6-001/tempDir-001/node3/.
[junit4] 2> 6020 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:50939_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1F377C89C3176F6-001/tempDir-001/node1/.
[junit4] 2> 6122 INFO (jetty-launcher-1-thread-4) [n:127.0.0.1:47037_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1F377C89C3176F6-001/tempDir-001/node4/.
[junit4] 2> 6245 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 6248 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42404/solr ready
[junit4] 2> 7804 WARN (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 9494 WARN (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 9941 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.m.log Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog
[junit4] 2> 9992 WARN (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 10144 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 10257 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_51394_hdfs____gyg5jw/webapp
[junit4] 2> 11670 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:51394
[junit4] 2> 13415 WARN (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 13420 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 13436 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_36164_datanode____.xihzu5/webapp
[junit4] 2> 13915 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36164
[junit4] 2> 16335 INFO (IPC Server handler 4 on 51007) [ ] BlockStateChange BLOCK* processReport: from storage DS-b8064d89-1898-4366-a483-32e8cf956eed node DatanodeRegistration(127.0.0.1:49158, datanodeUuid=b9f1bc5b-fca8-40aa-8f19-9a16cb00a028, infoPort=45162, infoSecurePort=0, ipcPort=52400, storageInfo=lv=-56;cid=testClusterID;nsid=1417652438;c=0), blocks: 0, hasStaleStorage: true, processing time: 3 msecs
[junit4] 2> 16336 INFO (IPC Server handler 4 on 51007) [ ] BlockStateChange BLOCK* processReport: from storage DS-a6d39e80-814f-429b-8781-9cecbe0c824f node DatanodeRegistration(127.0.0.1:49158, datanodeUuid=b9f1bc5b-fca8-40aa-8f19-9a16cb00a028, infoPort=45162, infoSecurePort=0, ipcPort=52400, storageInfo=lv=-56;cid=testClusterID;nsid=1417652438;c=0), blocks: 0, hasStaleStorage: false, processing time: 1 msecs
[junit4] 2> 16810 INFO (TEST-HdfsCollectionsAPIDistributedZkTest.testDeleteNonExistentCollection-seed#[1F377C89C3176F6]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDeleteNonExistentCollection
[junit4] 2> 16818 INFO (TEST-HdfsCollectionsAPIDistributedZkTest.testDeleteNonExistentCollection-seed#[1F377C89C3176F6]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 17427 INFO (qtp748027989-56) [n:127.0.0.1:56481_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=unknown_collection&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 17455 INFO (OverseerThreadFactory-18-thread-1-processing-n:127.0.0.1:47037_solr) [n:127.0.0.1:47037_solr ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
[junit4] 2> 17456 ERROR (OverseerThreadFactory-18-thread-1-processing-n:127.0.0.1:47037_solr) [n:127.0.0.1:47037_solr ] o.a.s.c.OverseerCollectionMessageHandler Collection: unknown_collection operation: delete failed:org.apache.solr.common.SolrException: Could not find collection : unknown_collection
[junit4] 2> at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:194)
[junit4] 2> at org.apache.solr.cloud.OverseerCollectionMessageHandler.collectionCmd(OverseerCollectionMessageHandler.java:846)
[junit4] 2> at org.apache.solr.cloud.DeleteCollectionCmd.call(DeleteCollectionCmd.java:88)
[junit4] 2> at org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:226)
[junit4] 2> at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:464)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 17549 INFO (qtp748027989-56) [n:127.0.0.1:56481_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=unknown_collection&action=DELETE&wt=javabin&version=2} status=400 QTime=172
[junit4] 2> 17615 INFO (qtp748027989-28) [n:127.0.0.1:56481_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&name=acollectionafterbaddelete&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 17646 INFO (OverseerCollectionConfigSetProcessor-97970750554832907-127.0.0.1:47037_solr-n_0000000000) [n:127.0.0.1:47037_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 17652 INFO (OverseerThreadFactory-18-thread-2-processing-n:127.0.0.1:47037_solr) [n:127.0.0.1:47037_solr ] o.a.s.c.CreateCollectionCmd Create collection acollectionafterbaddelete
[junit4] 2> 17881 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr ] o.a.s.u.TestInjection Inject random core creation delay of 5s
[junit4] 2> 17894 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr ] o.a.s.u.TestInjection Inject random core creation delay of 5s
[junit4] 2> 22881 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=acollectionafterbaddelete_shard1_replica2&action=CREATE&numShards=1&collection=acollectionafterbaddelete&shard=shard1&wt=javabin&version=2
[junit4] 2> 22887 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 22895 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=acollectionafterbaddelete_shard1_replica1&action=CREATE&numShards=1&collection=acollectionafterbaddelete&shard=shard1&wt=javabin&version=2
[junit4] 2> 22897 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 22904 INFO (zkCallback-23-thread-1-processing-n:127.0.0.1:50939_solr) [n:127.0.0.1:50939_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/acollectionafterbaddelete/state.json] for collection [acollectionafterbaddelete] has occurred - updating... (live nodes size: [4])
[junit4] 2> 22904 INFO (zkCallback-22-thread-1-processing-n:127.0.0.1:47037_solr) [n:127.0.0.1:47037_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/acollectionafterbaddelete/state.json] for collection [acollectionafterbaddelete] has occurred - updating... (live nodes size: [4])
[junit4] 2> 23015 INFO (zkCallback-23-thread-1-processing-n:127.0.0.1:50939_solr) [n:127.0.0.1:50939_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/acollectionafterbaddelete/state.json] for collection [acollectionafterbaddelete] has occurred - updating... (live nodes size: [4])
[junit4] 2> 23015 INFO (zkCallback-22-thread-1-processing-n:127.0.0.1:47037_solr) [n:127.0.0.1:47037_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/acollectionafterbaddelete/state.json] for collection [acollectionafterbaddelete] has occurred - updating... (live nodes size: [4])
[junit4] 2> 23985 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 23989 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 24118 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.s.IndexSchema [acollectionafterbaddelete_shard1_replica2] Schema name=minimal
[junit4] 2> 24127 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.s.IndexSchema [acollectionafterbaddelete_shard1_replica1] Schema name=minimal
[junit4] 2> 24234 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 24235 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 24235 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'acollectionafterbaddelete_shard1_replica1' using configuration from collection acollectionafterbaddelete, trusted=true
[junit4] 2> 24235 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'acollectionafterbaddelete_shard1_replica2' using configuration from collection acollectionafterbaddelete, trusted=true
[junit4] 2> 24262 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:51007/data
[junit4] 2> 24262 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 24263 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 24264 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:51007/data
[junit4] 2> 24265 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 24265 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 24270 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.SolrCore [[acollectionafterbaddelete_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1F377C89C3176F6-001/tempDir-001/node1/acollectionafterbaddelete_shard1_replica1], dataDir=[hdfs://localhost:51007/data/acollectionafterbaddelete/core_node2/data/]
[junit4] 2> 24270 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.SolrCore [[acollectionafterbaddelete_shard1_replica2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1F377C89C3176F6-001/tempDir-001/node4/acollectionafterbaddelete_shard1_replica2], dataDir=[hdfs://localhost:51007/data/acollectionafterbaddelete/core_node1/data/]
[junit4] 2> 24290 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ac59ebc
[junit4] 2> 24295 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ac59ebc
[junit4] 2> 24370 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51007/data/acollectionafterbaddelete/core_node1/data/snapshot_metadata
[junit4] 2> 24371 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51007/data/acollectionafterbaddelete/core_node2/data/snapshot_metadata
[junit4] 2> 24438 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 24438 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 24446 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
[junit4] 2> 24446 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 24446 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
[junit4] 2> 24446 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 25573 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 25576 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 25612 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51007/data/acollectionafterbaddelete/core_node2/data
[junit4] 2> 25624 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51007/data/acollectionafterbaddelete/core_node1/data
[junit4] 2> 25719 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51007/data/acollectionafterbaddelete/core_node2/data/index
[junit4] 2> 25758 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51007/data/acollectionafterbaddelete/core_node1/data/index
[junit4] 2> 25761 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 25761 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
[junit4] 2> 25761 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 25782 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 25783 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
[junit4] 2> 25783 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 25823 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 25827 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 26465 INFO (IPC Server handler 2 on 51007) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49158 is added to blk_1073741825_1001{UCState=COMMITTED, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b8064d89-1898-4366-a483-32e8cf956eed:NORMAL:127.0.0.1:49158|RBW]]} size 71
[junit4] 2> 26469 INFO (IPC Server handler 4 on 51007) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49158 is added to blk_1073741826_1002{UCState=COMMITTED, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b8064d89-1898-4366-a483-32e8cf956eed:NORMAL:127.0.0.1:49158|RBW]]} size 71
[junit4] 2> 28036 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 28036 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 28036 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 28042 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 28053 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 28070 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 28123 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 28124 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 28130 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 28135 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 28325 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6b6448df[acollectionafterbaddelete_shard1_replica1] main]
[junit4] 2> 28330 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@267095ea[acollectionafterbaddelete_shard1_replica2] main]
[junit4] 2> 28368 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 28375 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 28424 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 28425 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 28451 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 28454 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 28614 INFO (searcherExecutor-27-thread-1-processing-n:127.0.0.1:50939_solr x:acollectionafterbaddelete_shard1_replica1 s:shard1 c:acollectionafterbaddelete) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.SolrCore [acollectionafterbaddelete_shard1_replica1] Registered new searcher Searcher@6b6448df[acollectionafterbaddelete_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 28615 INFO (searcherExecutor-28-thread-1-processing-n:127.0.0.1:47037_solr x:acollectionafterbaddelete_shard1_replica2 s:shard1 c:acollectionafterbaddelete) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.SolrCore [acollectionafterbaddelete_shard1_replica2] Registered new searcher Searcher@267095ea[acollectionafterbaddelete_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 28670 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1567532037921832960
[junit4] 2> 28673 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1567532037926027264
[junit4] 2> 28727 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 28728 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 28730 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:47037/solr/acollectionafterbaddelete_shard1_replica2/
[junit4] 2> 28735 INFO (zkCallback-23-thread-1-processing-n:127.0.0.1:50939_solr) [n:127.0.0.1:50939_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/acollectionafterbaddelete/state.json] for collection [acollectionafterbaddelete] has occurred - updating... (live nodes size: [4])
[junit4] 2> 28735 INFO (zkCallback-22-thread-1-processing-n:127.0.0.1:47037_solr) [n:127.0.0.1:47037_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/acollectionafterbaddelete/state.json] for collection [acollectionafterbaddelete] has occurred - updating... (live nodes size: [4])
[junit4] 2> 28741 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=acollectionafterbaddelete_shard1_replica2 url=https://127.0.0.1:47037/solr START replicas=[https://127.0.0.1:50939/solr/acollectionafterbaddelete_shard1_replica1/] nUpdates=100
[junit4] 2> 28742 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=acollectionafterbaddelete_shard1_replica2 url=https://127.0.0.1:47037/solr DONE. We have no versions. sync failed.
[junit4] 2> 28785 INFO (qtp351731691-50) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 r:core_node2 x:acollectionafterbaddelete_shard1_replica1] o.a.s.c.S.Request [acollectionafterbaddelete_shard1_replica1] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=4
[junit4] 2> 28788 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 28788 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 28788 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 28811 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:47037/solr/acollectionafterbaddelete_shard1_replica2/ shard1
[junit4] 2> 28917 INFO (zkCallback-23-thread-1-processing-n:127.0.0.1:50939_solr) [n:127.0.0.1:50939_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/acollectionafterbaddelete/state.json] for collection [acollectionafterbaddelete] has occurred - updating... (live nodes size: [4])
[junit4] 2> 28917 INFO (zkCallback-22-thread-1-processing-n:127.0.0.1:47037_solr) [n:127.0.0.1:47037_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/acollectionafterbaddelete/state.json] for collection [acollectionafterbaddelete] has occurred - updating... (live nodes size: [4])
[junit4] 2> 28966 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 28971 INFO (qtp259325941-35) [n:127.0.0.1:47037_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=acollectionafterbaddelete_shard1_replica2&action=CREATE&numShards=1&collection=acollectionafterbaddelete&shard=shard1&wt=javabin&version=2} status=0 QTime=11093
[junit4] 2> 29079 INFO (zkCallback-23-thread-1-processing-n:127.0.0.1:50939_solr) [n:127.0.0.1:50939_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/acollectionafterbaddelete/state.json] for collection [acollectionafterbaddelete] has occurred - updating... (live nodes size: [4])
[junit4] 2> 29080 INFO (zkCallback-22-thread-1-processing-n:127.0.0.1:47037_solr) [n:127.0.0.1:47037_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/acollectionafterbaddelete/state.json] for collection [acollectionafterbaddelete] has occurred - updating... (live nodes size: [4])
[junit4] 2> 29738 INFO (qtp351731691-45) [n:127.0.0.1:50939_solr c:acollectionafterbaddelete s:shard1 x:acollectionafterbaddelete_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=acollectionafterbaddelete_shard1_replica1&action=CREATE&numShards=1&collection=acollectionafterbaddelete&shard=shard1&wt=javabin&version=2} status=0 QTime=11845
[junit4] 2> 29746 INFO (qtp748027989-28) [n:127.0.0.1:56481_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 29846 INFO (zkCallback-23-thread-1-processing-n:127.0.0.1:50939_solr) [n:127.0.0.1:50939_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/acollectionafterbaddelete/state.json] for collection [acollectionafterbaddelete] has occurred - updating... (live nodes size: [4])
[junit4] 2> 29846 INFO (zkCallback-22-thread-1-processing-n:127.0.0.1:47037_solr) [n:127.0.0.1:47037_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/acollectionafterbaddelete/state.json] for collection [acollectionafterbaddelete] has occurred - updating... (live nodes size: [4])
[junit4] 2> 30748 INFO (qtp748027989-28) [n:127.0.0.1:56481_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&name=acollectionafterbaddelete&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=13133
[junit4] 2> 30756 INFO (TEST-HdfsCollectionsAPIDistributedZkTest.testDeleteNonExistentCollection-seed#[1F377C89C3176F6]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDeleteNonExistentCollection
[junit4] 2> 30816 INFO (TEST-HdfsCollectionsAPIDistributedZkTest.testCoresAreDistributedAcrossNodes-seed#[1F377C89C3176F6]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testCoresAreDistributedAcrossNodes
[junit4] 2> 30818 INFO (TEST-HdfsCollectionsAPIDistributedZkTest.testCoresAreDistributedAcrossNodes-seed#[1F377C89C3176F6]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 30823 INFO (qtp748027989-44) [n:127.0.0.1:56481_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=acollectionafterbaddelete&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 30827 INFO (OverseerThreadFactory-18-thread-3-processing-n:127.0.0.1:47037_solr) [n:127.0.0.1:47037_solr ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
[junit4] 2> 30827 INFO (OverseerCollectionConfigSetProcessor-97970750554832907-127.0.0.1:47037_solr-n_0000000000) [n:127.0.0.1:47037_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 30835 INFO (qtp259325941-34) [n:127.0.0.1:47037_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.acollectionafterbaddelete.shard1.replica2
[junit4] 2> 30836 INFO (qtp259325941-34) [n:127.0.0.1:47037_solr ] o.a.s.c.SolrCore [acollectionafterbaddelete_shard1_replica2] CLOSING SolrCore org.apache.solr.core.SolrCore@5af8d969
[junit4] 2> 30840 INFO (qtp351731691-48) [n:127.0.0.1:50939_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.acollectionafterbaddelete.shard1.replica1
[junit4] 2> 30842 INFO (qtp351731691-48) [n:127.0.0.1:50939_solr ] o.a.s.c.SolrCore [acollectionafterbaddelete_shard1_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@581ed413
[junit4] 2> 31355 INFO (qtp259325941-34) [n:127.0.0.1:47037_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.acollectionafterbaddelete.shard1.replica2
[junit4] 2> 31397 INFO (qtp351731691-48) [n:127.0.0.1:50939_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.acollectionafterbaddelete.shard1.replica1
[junit4] 2> 31405 INFO (qtp351731691-48) [n:127.0.0.1:50939_solr ] o.a.s.s.h.HdfsDirectory Closing hdfs directory hdfs://localhost:51007/data/acollectionafterbaddelete/core_node2/data/snapshot_metadata
[junit4] 2> 31426 INFO (qtp351731691-48) [n:127.0.0.1:50939_solr ] o.a.s.s.h.HdfsDirectory Closing hdfs directory hdfs://localhost:51007/data/acollectionafterbaddelete/core_node2/data/index
[junit4] 2> 31427 INFO (qtp351731691-48) [n:127.0.0.1:50939_solr ] o.a.s.s.h.HdfsDirectory Closing hdfs directory hdfs://localhost:51007/data/acollectionafterbaddelete/core_node2/data
[junit4] 2> 31453 INFO (IPC Server handler 9 on 51007) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741826_1002 127.0.0.1:49158
[junit4] 2> 31458 INFO (qtp259325941-34) [n:127.0.0.1:47037_solr ] o.a.s.s.h.HdfsDirectory Closing hdfs directory hdfs://localhost:51007/data/acollectionafterbaddelete/core_node1/data/snapshot_metadata
[junit4] 2> 31462 INFO (qtp259325941-34) [n:127.0.0.1:47037_solr ] o.a.s.s.h.HdfsDirectory Closing hdfs directory hdfs://localhost:51007/data/acollectionafterbaddelete/core_node1/data/index
[junit4] 2> 31463 INFO (qtp259325941-34) [n:127.0.0.1:47037_solr ] o.a.s.s.h.HdfsDirectory Closing hdfs directory hdfs://localhost:51007/data/acollectionafterbaddelete/core_node1/data
[junit4] 2> 31483 INFO (IPC Server handler 1 on 51007) [ ] BlockStateChange BLOCK* addToInvalidates: blk_1073741825_1001 127.0.0.1:49158
[junit4] 2> 31505 INFO (qtp351731691-48) [n:127.0.0.1:50939_solr ] o.a.s.c.ShardLeaderElectionContextBase No version found for ephemeral leader parent node, won't remove previous leader registration.
[junit4] 2> 31508 WARN (zkCallback-23-thread-1-processing-n:127.0.0.1:50939_solr) [n:127.0.0.1:50939_solr ] o.a.s.c.LeaderElector Our node is no longer in line to be leader
[junit4] 2> 31523 INFO (qtp259325941-34) [n:127.0.0.1:47037_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=acollectionafterbaddelete_shard1_replica2&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=689
[junit4] 2> 31524 INFO (qtp351731691-48) [n:127.0.0.1:50939_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=acollectionafterbaddelete_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=684
[junit4] 2> 32234 INFO (qtp748027989-44) [n:127.0.0.1:56481_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=acollectionafterbaddelete&action=DELETE&wt=javabin&version=2} status=0 QTime=1411
[junit4] 2> 32241 INFO (qtp748027989-47) [n:127.0.0.1:56481_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&name=nodes_used_collection&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 32254 INFO (OverseerThreadFactory-18-thread-4-processing-n:127.0.0.1:47037_solr) [n:127.0.0.1:47037_solr ] o.a.s.c.CreateCollectionCmd Create collection nodes_used_collection
[junit4] 2> 32255 INFO (OverseerCollectionConfigSetProcessor-97970750554832907-127.0.0.1:47037_solr-n_0000000000) [n:127.0.0.1:47037_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 32469 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr ] o.a.s.u.TestInjection Inject random core creation delay of 5s
[junit4] 2> 32481 INFO (qtp351731691-49) [n:127.0.0.1:50939_solr ] o.a.s.u.TestInjection Inject random core creation delay of 5s
[junit4] 2> 32533 INFO (qtp748027989-28) [n:127.0.0.1:56481_solr ] o.a.s.u.TestInjection Inject random core creation delay of 5s
[junit4] 2> 32540 INFO (qtp1728100720-61) [n:127.0.0.1:38868_solr ] o.a.s.u.TestInjection Inject random core creation delay of 5s
[junit4] 2> 34151 INFO (org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@1e5ee701) [ ] BlockStateChange BLOCK* BlockManager: ask 127.0.0.1:49158 to delete [blk_1073741825_1001, blk_1073741826_1002]
[junit4] 2> 37470 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard1_replica2&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard1&wt=javabin&version=2
[junit4] 2> 37481 INFO (qtp351731691-49) [n:127.0.0.1:50939_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard2_replica2&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard2&wt=javabin&version=2
[junit4] 2> 37533 INFO (qtp748027989-28) [n:127.0.0.1:56481_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard2_replica1&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard2&wt=javabin&version=2
[junit4] 2> 37534 INFO (qtp748027989-28) [n:127.0.0.1:56481_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 37540 INFO (qtp1728100720-61) [n:127.0.0.1:38868_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard1_replica1&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard1&wt=javabin&version=2
[junit4] 2> 37541 INFO (qtp1728100720-61) [n:127.0.0.1:38868_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 37579 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 37614 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.s.IndexSchema [nodes_used_collection_shard1_replica2] Schema name=minimal
[junit4] 2> 37620 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 37620 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'nodes_used_collection_shard1_replica2' using configuration from collection nodes_used_collection, trusted=true
[junit4] 2> 37621 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:51007/data
[junit4] 2> 37621 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 37621 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 37621 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.SolrCore [[nodes_used_collection_shard1_replica2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1F377C89C3176F6-001/tempDir-001/node4/nodes_used_collection_shard1_replica2], dataDir=[hdfs://localhost:51007/data/nodes_used_collection/core_node1/data/]
[junit4] 2> 37623 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ac59ebc
[junit4] 2> 37623 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51007/data/nodes_used_collection/core_node1/data/snapshot_metadata
[junit4] 2> 37633 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 37634 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
[junit4] 2> 37634 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 37641 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 37643 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51007/data/nodes_used_collection/core_node1/data
[junit4] 2> 37647 INFO (zkCallback-23-thread-1-processing-n:127.0.0.1:50939_solr) [n:127.0.0.1:50939_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 37647 INFO (zkCallback-20-thread-1-processing-n:127.0.0.1:38868_solr) [n:127.0.0.1:38868_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 37647 INFO (zkCallback-21-thread-1-processing-n:127.0.0.1:56481_solr) [n:127.0.0.1:56481_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 37648 INFO (zkCallback-22-thread-1-processing-n:127.0.0.1:47037_solr) [n:127.0.0.1:47037_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 37695 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:51007/data/nodes_used_collection/core_node1/data/index
[junit4] 2> 37712 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 37712 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
[junit4] 2> 37712 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 37729 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 37850 INFO (IPC Server handler 4 on 51007) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49158 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-a6d39e80-814f-429b-8781-9cecbe0c824f:NORMAL:127.0.0.1:49158|RBW]]} size 0
[junit4] 2> 38047 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 38047 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 38047 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 38082 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 38083 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 38148 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@10c58559[nodes_used_collection_shard1_replica2] main]
[junit4] 2> 38150 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 38150 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 38151 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 38153 INFO (searcherExecutor-37-thread-1-processing-n:127.0.0.1:47037_solr x:nodes_used_collection_shard1_replica2 s:shard1 c:nodes_used_collection) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.SolrCore [nodes_used_collection_shard1_replica2] Registered new searcher Searcher@10c58559[nodes_used_collection_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 38160 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1567532047873867776
[junit4] 2> 38171 INFO (qtp259325941-29) [n:127.0.0.1:47037_solr c:nodes_used_collection s:shard1 x:nodes_used_collection_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 38275 INFO (zkCallback-20-thread-1-processing-n:127.0.0.1:38868_solr) [n:127.0.0.1:38868_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 38276 INFO (zkCallback-21-thread-1-processing-n:127.0.0.1:56481_solr) [n:127.0.0.1:56481_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 38276 INFO (zkCallback-22-thread-1-processing-n:127.0.0.1:47037_solr) [n:127.0.0.1:47037_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 38275 INFO (zkCallback-23-thread-1-processing-n:127.0.0.1:50939_solr) [n:127.0.0.1:50939_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 38541 INFO (qtp351731691-49) [n:127.0.0.1:50939_solr c:nodes_used_collection s:shard2 x:nodes_used_collection_shard2_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 38565 INFO (qtp351731691-49) [n:127.0.0.1:50939_solr c:nodes_used_collection s:shard2 x:nodes_used_collection_shard2_replica2] o.a.s.s.IndexSchema [nodes_used_collection_shard2_replica2] Schema name=minimal
[junit4] 2> 38573 INFO (qtp351731691-49) [n:127.0.0.1:50939_solr c:nodes_used_collection s:shard2 x:nodes_used_collection_shard2_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 38573 INFO (qtp351731691-49) [n:127.0.0.1:50939_solr c:nodes_used_collection s:shard2 x:nodes_used_collection_shard2_replica2] o.a.s.c.CoreContainer Creating SolrCore 'nodes_used_collection_shard2_replica2' using configuration from collection nodes_used_collection, trusted=true
[junit4] 2> 38574 INFO (qtp351731691-49) [n:127.0.0.1:50939_solr c:nodes_used_collection s:shard2 x:nodes_used_collection_shard2_replica2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:51007/data
[junit4] 2> 38622 INFO (qtp351731691-49) [n:127.0.0.1:50939_solr c:nodes_used_collection s:shard2 x:nodes_used_collection_shard2_replica2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 38622 INFO (qtp351731691-49) [n:127.0.0.1:50939_solr c:nodes_used_collection s:shard2 x:nodes_used_collection_shard2_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 38622 INFO (qtp351731691-49) [n:127.0.0.1:50939_solr c:nodes_used_collection s:shard2 x:nodes_used_collection_shard2_replica2] o.a.s.c.SolrCore [[nodes_used_collection_shard2_replica2] ] Opening new SolrCore at [/x1/jenkins/
[...truncated too long message...]
ir-002/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1F377C89C3176F6-001/tempDir-002/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:51007) [ ] o.a.h.h.s.d.DataNode BPOfferService for Block pool BP-710556347-10.41.0.5-1494915024690 (Datanode Uuid b9f1bc5b-fca8-40aa-8f19-9a16cb00a028) service to localhost/127.0.0.1:51007 interrupted
[junit4] 2> 234858 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1F377C89C3176F6-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1F377C89C3176F6-001/tempDir-002/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:51007) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-710556347-10.41.0.5-1494915024690 (Datanode Uuid b9f1bc5b-fca8-40aa-8f19-9a16cb00a028) service to localhost/127.0.0.1:51007
[junit4] 2> 234886 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 234888 ERROR (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.a.h.m.l.MethodMetric Error invoking method getBlocksTotal
[junit4] 2> java.lang.reflect.InvocationTargetException
[junit4] 2> at sun.reflect.GeneratedMethodAccessor44.invoke(Unknown Source)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MethodMetric$2.snapshot(MethodMetric.java:111)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MethodMetric.snapshot(MethodMetric.java:144)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MetricsRegistry.snapshot(MetricsRegistry.java:401)
[junit4] 2> at org.apache.hadoop.metrics2.lib.MetricsSourceBuilder$1.getMetrics(MetricsSourceBuilder.java:79)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
[junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
[junit4] 2> at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:203)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.teardownClass(HdfsCollectionsAPIDistributedZkTest.java:77)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 53 more
[junit4] 2> 234891 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:42404 42404
[junit4] 2> 234892 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1F377C89C3176F6]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1F377C89C3176F6-001
[junit4] 2> May 16, 2017 6:14:10 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 19 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {_version_=PostingsFormat(name=Memory doPackFST= true), id=PostingsFormat(name=LuceneFixedGap)}, docValues:{}, maxPointsInLeafNode=1839, maxMBSortInHeap=5.499579560460535, sim=RandomSimilarity(queryNorm=true,coord=no): {}, locale=lt-LT, timezone=Africa/Khartoum
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=4,threads=6,free=172563632,total=403177472
[junit4] 2> NOTE: All tests run in this JVM: [TestJavabinTupleStreamParser, HdfsCollectionsAPIDistributedZkTest]
[junit4] Completed [39/711 (1!)] on J1 in 244.26s, 21 tests, 1 failure <<< FAILURES!
[...truncated 9614 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/build.xml:783: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/build.xml:727: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid1694.hprof
Total time: 345 minutes 39 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-NightlyTests-6.x - Build # 349 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.x/349/
1 tests failed.
FAILED: org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test
Error Message:
Error from server at http://127.0.0.1:34435/ic_: Error CREATEing SolrCore 'test_unload_shard_and_collection_1': Unable to create core [test_unload_shard_and_collection_1] Caused by: Direct buffer memory
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:34435/ic_: Error CREATEing SolrCore 'test_unload_shard_and_collection_1': Unable to create core [test_unload_shard_and_collection_1] Caused by: Direct buffer memory
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:610)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:279)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268)
at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
at org.apache.solr.cloud.UnloadDistributedZkTest.testUnloadShardAndCollection(UnloadDistributedZkTest.java:125)
at org.apache.solr.cloud.UnloadDistributedZkTest.test(UnloadDistributedZkTest.java:70)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 13336 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/init-core-data-001
[junit4] 2> 3726173 WARN (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=42 numCloses=42
[junit4] 2> 3726173 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields
[junit4] 2> 3726175 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 3726175 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /ic_/
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 3726207 WARN (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 3726212 WARN (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3726214 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3726230 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_57959_hdfs____.vp5ej/webapp
[junit4] 2> 3726620 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:57959
[junit4] 2> 3726937 WARN (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3726938 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3726954 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_36955_datanode____re1nwd/webapp
[junit4] 2> 3732303 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36955
[junit4] 2> 3732348 WARN (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3732349 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3732365 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_49905_datanode____.4s8g1s/webapp
[junit4] 2> 3732415 INFO (IPC Server handler 6 on 48591) [ ] BlockStateChange BLOCK* processReport: from storage DS-ce639043-c04c-403f-8883-c807bd7366d6 node DatanodeRegistration(127.0.0.1:56594, datanodeUuid=1680047b-8f9c-4ee2-bb63-9bd562e5ae60, infoPort=47520, infoSecurePort=0, ipcPort=35922, storageInfo=lv=-56;cid=testClusterID;nsid=942609280;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 3732415 INFO (IPC Server handler 6 on 48591) [ ] BlockStateChange BLOCK* processReport: from storage DS-875aa893-cb7a-4a64-b8b7-a92cd4aa2086 node DatanodeRegistration(127.0.0.1:56594, datanodeUuid=1680047b-8f9c-4ee2-bb63-9bd562e5ae60, infoPort=47520, infoSecurePort=0, ipcPort=35922, storageInfo=lv=-56;cid=testClusterID;nsid=942609280;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 3732771 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:49905
[junit4] 2> 3732856 INFO (IPC Server handler 1 on 48591) [ ] BlockStateChange BLOCK* processReport: from storage DS-547ecbf4-62fa-4a24-9655-c4d7e0815c6c node DatanodeRegistration(127.0.0.1:49387, datanodeUuid=3bec16e4-d7d4-4d60-8530-7efb56557320, infoPort=42007, infoSecurePort=0, ipcPort=43674, storageInfo=lv=-56;cid=testClusterID;nsid=942609280;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 3732856 INFO (IPC Server handler 1 on 48591) [ ] BlockStateChange BLOCK* processReport: from storage DS-7f84de8f-4743-44ed-a2f1-dd0158fc8a7f node DatanodeRegistration(127.0.0.1:49387, datanodeUuid=3bec16e4-d7d4-4d60-8530-7efb56557320, infoPort=42007, infoSecurePort=0, ipcPort=43674, storageInfo=lv=-56;cid=testClusterID;nsid=942609280;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 3732950 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3732950 INFO (Thread-21982) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3732950 INFO (Thread-21982) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3732952 ERROR (Thread-21982) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3733050 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.ZkTestServer start zk server on port:41936
[junit4] 2> 3733076 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 3733079 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 3733081 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 3733082 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 3733084 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 3733086 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 3733087 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 3733089 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 3733090 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 3733093 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 3733094 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 3733211 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/control-001/cores/collection1
[junit4] 2> 3733213 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3733214 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@ab8a08e{/ic_,null,AVAILABLE}
[junit4] 2> 3733214 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3382391a{HTTP/1.1,[http/1.1]}{127.0.0.1:60671}
[junit4] 2> 3733215 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.e.j.s.Server Started @3736649ms
[junit4] 2> 3733215 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:48591/hdfs__localhost_48591__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-6.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001_tempDir-002_control_data, hostContext=/ic_, hostPort=60671, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/control-001/cores}
[junit4] 2> 3733216 ERROR (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3733216 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.7.0
[junit4] 2> 3733216 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3733216 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3733216 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-15T03:22:46.329Z
[junit4] 2> 3733219 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3733219 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/control-001/solr.xml
[junit4] 2> 3733225 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 3733226 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41936/solr
[junit4] 2> 3733249 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:60671_ic_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3733250 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:60671_ic_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60671_ic_
[junit4] 2> 3733252 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:60671_ic_ ] o.a.s.c.Overseer Overseer (id=97964429545570308-127.0.0.1:60671_ic_-n_0000000000) starting
[junit4] 2> 3733259 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:60671_ic_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60671_ic_
[junit4] 2> 3733262 INFO (zkCallback-3678-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3733389 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:60671_ic_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/control-001/cores
[junit4] 2> 3733389 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:60671_ic_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 3733391 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3733393 INFO (OverseerStateUpdate-97964429545570308-127.0.0.1:60671_ic_-n_0000000000) [n:127.0.0.1:60671_ic_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 3733413 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 3733439 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 3733573 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3733589 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection, trusted=true
[junit4] 2> 3733589 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:48591/solr_hdfs_home
[junit4] 2> 3733589 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 3733589 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3733589 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/control-001/cores/collection1], dataDir=[hdfs://localhost:48591/solr_hdfs_home/control_collection/core_node1/data/]
[junit4] 2> 3733590 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4256d7d1
[junit4] 2> 3733590 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48591/solr_hdfs_home/control_collection/core_node1/data/snapshot_metadata
[junit4] 2> 3733598 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3733598 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 3733598 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3733606 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3733607 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48591/solr_hdfs_home/control_collection/core_node1/data
[junit4] 2> 3733623 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48591/solr_hdfs_home/control_collection/core_node1/data/index
[junit4] 2> 3733630 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3733630 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 3733630 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3733637 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3733637 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=77.0234375, floorSegmentMB=0.5517578125, forceMergeDeletesPctAllowed=25.215395839543344, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.40998683821439375
[junit4] 2> 3733649 INFO (IPC Server handler 6 on 48591) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49387 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-875aa893-cb7a-4a64-b8b7-a92cd4aa2086:NORMAL:127.0.0.1:56594|RBW], ReplicaUC[[DISK]DS-547ecbf4-62fa-4a24-9655-c4d7e0815c6c:NORMAL:127.0.0.1:49387|FINALIZED]]} size 0
[junit4] 2> 3733650 INFO (IPC Server handler 8 on 48591) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:56594 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-875aa893-cb7a-4a64-b8b7-a92cd4aa2086:NORMAL:127.0.0.1:56594|RBW], ReplicaUC[[DISK]DS-547ecbf4-62fa-4a24-9655-c4d7e0815c6c:NORMAL:127.0.0.1:49387|FINALIZED]]} size 0
[junit4] 2> 3733657 WARN (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3733698 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 3733698 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3733698 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 3733708 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3733709 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3733710 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=73.9892578125, floorSegmentMB=1.763671875, forceMergeDeletesPctAllowed=5.83586190947377, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.37596782675488416
[junit4] 2> 3733714 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2dc2a4d8[collection1] main]
[junit4] 2> 3733715 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3733715 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3733715 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 3733717 INFO (searcherExecutor-8779-thread-1-processing-n:127.0.0.1:60671_ic_ x:collection1 c:control_collection) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2dc2a4d8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3733719 WARN (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:57142/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:57142 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3733719 WARN (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:57142/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:57142 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3733719 INFO (coreLoadExecutor-8778-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1567430873534431232
[junit4] 2> 3733726 INFO (coreZkRegister-8771-thread-1-processing-n:127.0.0.1:60671_ic_ x:collection1 c:control_collection) [n:127.0.0.1:60671_ic_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3733726 INFO (coreZkRegister-8771-thread-1-processing-n:127.0.0.1:60671_ic_ x:collection1 c:control_collection) [n:127.0.0.1:60671_ic_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3733726 INFO (coreZkRegister-8771-thread-1-processing-n:127.0.0.1:60671_ic_ x:collection1 c:control_collection) [n:127.0.0.1:60671_ic_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60671/ic_/collection1/
[junit4] 2> 3733726 INFO (coreZkRegister-8771-thread-1-processing-n:127.0.0.1:60671_ic_ x:collection1 c:control_collection) [n:127.0.0.1:60671_ic_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3733726 INFO (coreZkRegister-8771-thread-1-processing-n:127.0.0.1:60671_ic_ x:collection1 c:control_collection) [n:127.0.0.1:60671_ic_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:60671/ic_/collection1/ has no replicas
[junit4] 2> 3733726 INFO (coreZkRegister-8771-thread-1-processing-n:127.0.0.1:60671_ic_ x:collection1 c:control_collection) [n:127.0.0.1:60671_ic_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3733730 INFO (coreZkRegister-8771-thread-1-processing-n:127.0.0.1:60671_ic_ x:collection1 c:control_collection) [n:127.0.0.1:60671_ic_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60671/ic_/collection1/ shard1
[junit4] 2> 3733881 INFO (coreZkRegister-8771-thread-1-processing-n:127.0.0.1:60671_ic_ x:collection1 c:control_collection) [n:127.0.0.1:60671_ic_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3733902 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3733904 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41936/solr ready
[junit4] 2> 3733904 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 3734026 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-1-001/cores/collection1
[junit4] 2> 3734026 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-1-001
[junit4] 2> 3734027 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3734029 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2dbb091e{/ic_,null,AVAILABLE}
[junit4] 2> 3734030 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@69d88d07{HTTP/1.1,[http/1.1]}{127.0.0.1:34435}
[junit4] 2> 3734030 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.e.j.s.Server Started @3737464ms
[junit4] 2> 3734031 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:48591/hdfs__localhost_48591__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-6.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001_tempDir-002_jetty1, solrconfig=solrconfig.xml, hostContext=/ic_, hostPort=34435, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-1-001/cores}
[junit4] 2> 3734031 ERROR (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3734031 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.7.0
[junit4] 2> 3734031 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3734031 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3734031 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-15T03:22:47.144Z
[junit4] 2> 3734034 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3734034 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-1-001/solr.xml
[junit4] 2> 3734041 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 3734042 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41936/solr
[junit4] 2> 3734052 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:34435_ic_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3734054 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:34435_ic_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3734058 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:34435_ic_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34435_ic_
[junit4] 2> 3734059 INFO (zkCallback-3682-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3734060 INFO (zkCallback-3687-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3734060 INFO (zkCallback-3678-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3734144 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:34435_ic_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-1-001/cores
[junit4] 2> 3734144 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:34435_ic_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 3734145 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3734147 INFO (OverseerStateUpdate-97964429545570308-127.0.0.1:60671_ic_-n_0000000000) [n:127.0.0.1:60671_ic_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
[junit4] 2> 3735167 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 3735182 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 3735313 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3735325 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1, trusted=true
[junit4] 2> 3735325 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:48591/solr_hdfs_home
[junit4] 2> 3735325 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 3735325 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3735325 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-1-001/cores/collection1], dataDir=[hdfs://localhost:48591/solr_hdfs_home/collection1/core_node1/data/]
[junit4] 2> 3735326 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4256d7d1
[junit4] 2> 3735326 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48591/solr_hdfs_home/collection1/core_node1/data/snapshot_metadata
[junit4] 2> 3735334 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3735334 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 3735334 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3735340 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3735341 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48591/solr_hdfs_home/collection1/core_node1/data
[junit4] 2> 3735357 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48591/solr_hdfs_home/collection1/core_node1/data/index
[junit4] 2> 3735364 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3735364 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 3735364 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3735370 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3735370 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=77.0234375, floorSegmentMB=0.5517578125, forceMergeDeletesPctAllowed=25.215395839543344, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.40998683821439375
[junit4] 2> 3735380 INFO (IPC Server handler 2 on 48591) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49387 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-875aa893-cb7a-4a64-b8b7-a92cd4aa2086:NORMAL:127.0.0.1:56594|RBW], ReplicaUC[[DISK]DS-7f84de8f-4743-44ed-a2f1-dd0158fc8a7f:NORMAL:127.0.0.1:49387|RBW]]} size 0
[junit4] 2> 3735381 INFO (IPC Server handler 5 on 48591) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:56594 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-7f84de8f-4743-44ed-a2f1-dd0158fc8a7f:NORMAL:127.0.0.1:49387|RBW], ReplicaUC[[DISK]DS-ce639043-c04c-403f-8883-c807bd7366d6:NORMAL:127.0.0.1:56594|FINALIZED]]} size 0
[junit4] 2> 3735388 WARN (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3735435 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 3735435 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3735435 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 3735446 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3735446 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3735448 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=73.9892578125, floorSegmentMB=1.763671875, forceMergeDeletesPctAllowed=5.83586190947377, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.37596782675488416
[junit4] 2> 3735452 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2dee5d5e[collection1] main]
[junit4] 2> 3735453 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3735453 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3735453 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 3735455 INFO (searcherExecutor-8790-thread-1-processing-n:127.0.0.1:34435_ic_ x:collection1 c:collection1) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2dee5d5e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3735457 WARN (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:57142/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:57142 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3735457 WARN (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:57142/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:57142 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3735458 WARN (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:57142/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:57142 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3735458 INFO (coreLoadExecutor-8789-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1567430875357904896
[junit4] 2> 3735464 INFO (coreZkRegister-8784-thread-1-processing-n:127.0.0.1:34435_ic_ x:collection1 c:collection1) [n:127.0.0.1:34435_ic_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3735465 INFO (coreZkRegister-8784-thread-1-processing-n:127.0.0.1:34435_ic_ x:collection1 c:collection1) [n:127.0.0.1:34435_ic_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3735465 INFO (coreZkRegister-8784-thread-1-processing-n:127.0.0.1:34435_ic_ x:collection1 c:collection1) [n:127.0.0.1:34435_ic_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:34435/ic_/collection1/
[junit4] 2> 3735465 INFO (coreZkRegister-8784-thread-1-processing-n:127.0.0.1:34435_ic_ x:collection1 c:collection1) [n:127.0.0.1:34435_ic_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3735465 INFO (coreZkRegister-8784-thread-1-processing-n:127.0.0.1:34435_ic_ x:collection1 c:collection1) [n:127.0.0.1:34435_ic_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:34435/ic_/collection1/ has no replicas
[junit4] 2> 3735465 INFO (coreZkRegister-8784-thread-1-processing-n:127.0.0.1:34435_ic_ x:collection1 c:collection1) [n:127.0.0.1:34435_ic_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3735468 INFO (coreZkRegister-8784-thread-1-processing-n:127.0.0.1:34435_ic_ x:collection1 c:collection1) [n:127.0.0.1:34435_ic_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:34435/ic_/collection1/ shard2
[junit4] 2> 3735620 INFO (coreZkRegister-8784-thread-1-processing-n:127.0.0.1:34435_ic_ x:collection1 c:collection1) [n:127.0.0.1:34435_ic_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3735764 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-2-001/cores/collection1
[junit4] 2> 3735764 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-2-001
[junit4] 2> 3735765 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3735766 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@466cdb6{/ic_,null,AVAILABLE}
[junit4] 2> 3735767 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@38e16fcc{HTTP/1.1,[http/1.1]}{127.0.0.1:47636}
[junit4] 2> 3735767 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.e.j.s.Server Started @3739200ms
[junit4] 2> 3735767 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:48591/hdfs__localhost_48591__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-6.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001_tempDir-002_jetty2, solrconfig=solrconfig.xml, hostContext=/ic_, hostPort=47636, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-2-001/cores}
[junit4] 2> 3735767 ERROR (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3735767 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.7.0
[junit4] 2> 3735767 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3735767 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3735768 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-15T03:22:48.881Z
[junit4] 2> 3735787 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3735787 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-2-001/solr.xml
[junit4] 2> 3735794 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 3735795 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41936/solr
[junit4] 2> 3735810 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:47636_ic_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3735814 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:47636_ic_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3735816 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:47636_ic_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47636_ic_
[junit4] 2> 3735818 INFO (zkCallback-3682-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3735818 INFO (zkCallback-3687-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3735818 INFO (zkCallback-3693-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3735822 INFO (zkCallback-3678-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3735977 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:47636_ic_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-2-001/cores
[junit4] 2> 3735977 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:47636_ic_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 3735977 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3735979 INFO (OverseerStateUpdate-97964429545570308-127.0.0.1:60671_ic_-n_0000000000) [n:127.0.0.1:60671_ic_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 3736998 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 3737013 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 3737134 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3737149 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1, trusted=true
[junit4] 2> 3737150 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:48591/solr_hdfs_home
[junit4] 2> 3737150 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 3737150 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3737150 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-2-001/cores/collection1], dataDir=[hdfs://localhost:48591/solr_hdfs_home/collection1/core_node2/data/]
[junit4] 2> 3737150 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4256d7d1
[junit4] 2> 3737151 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48591/solr_hdfs_home/collection1/core_node2/data/snapshot_metadata
[junit4] 2> 3737159 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3737159 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 3737159 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3737165 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3737166 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48591/solr_hdfs_home/collection1/core_node2/data
[junit4] 2> 3737182 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48591/solr_hdfs_home/collection1/core_node2/data/index
[junit4] 2> 3737189 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3737189 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 3737189 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3737196 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3737196 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=77.0234375, floorSegmentMB=0.5517578125, forceMergeDeletesPctAllowed=25.215395839543344, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.40998683821439375
[junit4] 2> 3737204 INFO (IPC Server handler 5 on 48591) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:56594 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-7f84de8f-4743-44ed-a2f1-dd0158fc8a7f:NORMAL:127.0.0.1:49387|RBW], ReplicaUC[[DISK]DS-875aa893-cb7a-4a64-b8b7-a92cd4aa2086:NORMAL:127.0.0.1:56594|FINALIZED]]} size 0
[junit4] 2> 3737204 INFO (IPC Server handler 7 on 48591) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49387 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-875aa893-cb7a-4a64-b8b7-a92cd4aa2086:NORMAL:127.0.0.1:56594|FINALIZED], ReplicaUC[[DISK]DS-547ecbf4-62fa-4a24-9655-c4d7e0815c6c:NORMAL:127.0.0.1:49387|FINALIZED]]} size 0
[junit4] 2> 3737212 WARN (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3737258 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 3737258 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3737258 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 3737268 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3737268 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3737270 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=73.9892578125, floorSegmentMB=1.763671875, forceMergeDeletesPctAllowed=5.83586190947377, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.37596782675488416
[junit4] 2> 3737274 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6477ad99[collection1] main]
[junit4] 2> 3737275 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3737276 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3737276 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 3737278 INFO (searcherExecutor-8801-thread-1-processing-n:127.0.0.1:47636_ic_ x:collection1 c:collection1) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6477ad99[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3737280 WARN (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:57142/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:57142 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3737280 WARN (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:57142/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:57142 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3737281 WARN (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:57142/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:57142 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3737281 INFO (coreLoadExecutor-8800-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1567430877269458944
[junit4] 2> 3737289 INFO (coreZkRegister-8795-thread-1-processing-n:127.0.0.1:47636_ic_ x:collection1 c:collection1) [n:127.0.0.1:47636_ic_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3737289 INFO (coreZkRegister-8795-thread-1-processing-n:127.0.0.1:47636_ic_ x:collection1 c:collection1) [n:127.0.0.1:47636_ic_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3737289 INFO (coreZkRegister-8795-thread-1-processing-n:127.0.0.1:47636_ic_ x:collection1 c:collection1) [n:127.0.0.1:47636_ic_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:47636/ic_/collection1/
[junit4] 2> 3737289 INFO (coreZkRegister-8795-thread-1-processing-n:127.0.0.1:47636_ic_ x:collection1 c:collection1) [n:127.0.0.1:47636_ic_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3737290 INFO (coreZkRegister-8795-thread-1-processing-n:127.0.0.1:47636_ic_ x:collection1 c:collection1) [n:127.0.0.1:47636_ic_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:47636/ic_/collection1/ has no replicas
[junit4] 2> 3737290 INFO (coreZkRegister-8795-thread-1-processing-n:127.0.0.1:47636_ic_ x:collection1 c:collection1) [n:127.0.0.1:47636_ic_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3737293 INFO (coreZkRegister-8795-thread-1-processing-n:127.0.0.1:47636_ic_ x:collection1 c:collection1) [n:127.0.0.1:47636_ic_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:47636/ic_/collection1/ shard1
[junit4] 2> 3737444 INFO (coreZkRegister-8795-thread-1-processing-n:127.0.0.1:47636_ic_ x:collection1 c:collection1) [n:127.0.0.1:47636_ic_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3737601 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-3-001/cores/collection1
[junit4] 2> 3737602 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-3-001
[junit4] 2> 3737612 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3737616 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@16ffd25a{/ic_,null,AVAILABLE}
[junit4] 2> 3737616 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2baf3c77{HTTP/1.1,[http/1.1]}{127.0.0.1:55138}
[junit4] 2> 3737616 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.e.j.s.Server Started @3741050ms
[junit4] 2> 3737616 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:48591/hdfs__localhost_48591__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-6.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001_tempDir-002_jetty3, solrconfig=solrconfig.xml, hostContext=/ic_, hostPort=55138, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-3-001/cores}
[junit4] 2> 3737617 ERROR (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3737617 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.7.0
[junit4] 2> 3737617 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3737617 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3737617 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-15T03:22:50.730Z
[junit4] 2> 3737626 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3737626 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-3-001/solr.xml
[junit4] 2> 3737633 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 3737634 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41936/solr
[junit4] 2> 3737658 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:55138_ic_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 3737661 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:55138_ic_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3737663 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:55138_ic_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55138_ic_
[junit4] 2> 3737664 INFO (zkCallback-3682-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3737665 INFO (zkCallback-3699-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3737665 INFO (zkCallback-3687-thread-1-processing-n:127.0.0.1:34435_ic_) [n:127.0.0.1:34435_ic_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3737665 INFO (zkCallback-3678-thread-1-processing-n:127.0.0.1:60671_ic_) [n:127.0.0.1:60671_ic_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3737665 INFO (zkCallback-3693-thread-1-processing-n:127.0.0.1:47636_ic_) [n:127.0.0.1:47636_ic_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3737791 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:55138_ic_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-3-001/cores
[junit4] 2> 3737791 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[FA9A28B99A1BE603]) [n:127.0.0.1:55138_ic_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 3737792 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3737793 INFO (OverseerStateUpdate-97964429545570308-127.0.0.1:60671_ic_-n_0000000000) [n:127.0.0.1:60671_ic_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
[junit4] 2> 3738810 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 3738849 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 3738973 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3738987 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1, trusted=true
[junit4] 2> 3738987 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:48591/solr_hdfs_home
[junit4] 2> 3738987 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 3738987 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3738987 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001/shard-3-001/cores/collection1], dataDir=[hdfs://localhost:48591/solr_hdfs_home/collection1/core_node3/data/]
[junit4] 2> 3738988 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4256d7d1
[junit4] 2> 3738988 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48591/solr_hdfs_home/collection1/core_node3/data/snapshot_metadata
[junit4] 2> 3738996 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3738996 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 3738996 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3739014 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3739015 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48591/solr_hdfs_home/collection1/core_node3/data
[junit4] 2> 3739032 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:48591/solr_hdfs_home/collection1/core_node3/data/index
[junit4] 2> 3739039 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3739039 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 3739039 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3739047 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3739048 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=77.0234375, floorSegmentMB=0.5517578125, forceMergeDeletesPctAllowed=25.215395839543344, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.40998683821439375
[junit4] 2> 3739059 INFO (IPC Server handler 7 on 48591) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:49387 is added to blk_1073741828_1004{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-875aa893-cb7a-4a64-b8b7-a92cd4aa2086:NORMAL:127.0.0.1:56594|RBW], ReplicaUC[[DISK]DS-7f84de8f-4743-44ed-a2f1-dd0158fc8a7f:NORMAL:127.0.0.1:49387|FINALIZED]]} size 0
[junit4] 2> 3739061 INFO (IPC Server handler 6 on 48591) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:56594 is added to blk_1073741828_1004{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-7f84de8f-4743-44ed-a2f1-dd0158fc8a7f:NORMAL:127.0.0.1:49387|FINALIZED], ReplicaUC[[DISK]DS-ce639043-c04c-403f-8883-c807bd7366d6:NORMAL:127.0.0.1:56594|FINALIZED]]} size 0
[junit4] 2> 3739072 WARN (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3739122 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 3739123 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3739123 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 3739134 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3739134 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3739136 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=73.9892578125, floorSegmentMB=1.763671875, forceMergeDeletesPctAllowed=5.83586190947377, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.37596782675488416
[junit4] 2> 3739141 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@340e4295[collection1] main]
[junit4] 2> 3739142 INFO (coreLoadExecutor-8811-thread-1-processing-n:127.0.0.1:55138_ic_) [n:127.0.0.1:55138_ic_ c:collection1 x:collection1] o.a.s.r.Manag
[...truncated too long message...]
t4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
[junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
[junit4] 2> at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:203)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.teardownClass(HdfsUnloadDistributedZkTest.java:46)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 53 more
[junit4] 2> 3787212 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[FA9A28B99A1BE603]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_FA9A28B99A1BE603-001
[junit4] 2> May 15, 2017 3:23:40 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62), sim=RandomSimilarity(queryNorm=false,coord=no): {}, locale=id, timezone=America/Rainy_River
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=4,threads=2,free=242953688,total=532676608
[junit4] 2> NOTE: All tests run in this JVM: [TestCSVResponseWriter, RemoteQueryErrorTest, LargeFieldTest, TestQuerySenderListener, DistributedDebugComponentTest, SpellCheckCollatorTest, TestSearchPerf, TestDistributedStatsComponentCardinality, DirectSolrSpellCheckerTest, HdfsThreadLeakTest, TestComplexPhraseLeadingWildcard, CdcrBootstrapTest, TestTrackingShardHandlerFactory, DocumentBuilderTest, TestConfigOverlay, DeleteInactiveReplicaTest, SpatialHeatmapFacetsTest, ConjunctionSolrSpellCheckerTest, JsonLoaderTest, TestLockTree, TestRawTransformer, ClassificationUpdateProcessorTest, SampleTest, TestHalfAndHalfDocValues, WordBreakSolrSpellCheckerTest, BlockJoinFacetRandomTest, DocValuesTest, TestSolrCLIRunExample, IndexSchemaRuntimeFieldTest, TestCloudDeleteByQuery, TestBM25SimilarityFactory, DistribCursorPagingTest, TestExactSharedStatsCache, TestSortByMinMaxFunction, SolrCloudExampleTest, TestFieldCacheVsDocValues, TestReqParamsAPI, RAMDirectoryFactoryTest, SolrCoreMetricManagerTest, TestStressVersions, HighlighterTest, TestSimpleTrackingShardHandler, SoftAutoCommitTest, SpellCheckComponentTest, ChangedSchemaMergeTest, OpenCloseCoreStressTest, TestFieldCacheSanityChecker, ConvertedLegacyTest, SolrCmdDistributorTest, TestLuceneMatchVersion, TestSolrIndexConfig, BasicAuthIntegrationTest, LeaderFailoverAfterPartitionTest, UpdateRequestProcessorFactoryTest, ZkSolrClientTest, ShardRoutingTest, TestSchemaVersionResource, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestTolerantUpdateProcessorCloud, MultiTermTest, SpellingQueryConverterTest, IndexBasedSpellCheckerTest, TestManagedSchemaThreadSafety, TestDocTermOrds, AssignTest, TermVectorComponentTest, DateMathParserTest, DistributedFacetPivotSmallTest, TestSchemalessBufferedUpdates, HdfsRecoveryZkTest, TestManagedResource, CoreAdminRequestStatusTest, BasicFunctionalityTest, TestCursorMarkWithoutUniqueKey, VersionInfoTest, TestAuthorizationFramework, TestFiltering, SaslZkACLProviderTest, BigEndianAscendingWordDeserializerTest, SolrIndexConfigTest, CoreSorterTest, TestJsonFacets, TestDynamicFieldCollectionResource, TestMacroExpander, EchoParamsTest, TestDynamicLoading, TestFieldResource, ClusterStateUpdateTest, SolrIndexSplitterTest, SubstringBytesRefFilterTest, TestManagedSynonymFilterFactory, TolerantUpdateProcessorTest, TestSolrCloudWithHadoopAuthPlugin, SolrMetricsIntegrationTest, URLClassifyProcessorTest, TestRTGBase, HdfsDirectoryFactoryTest, TestScoreJoinQPScore, TestRandomRequestDistribution, ZkStateReaderTest, SchemaVersionSpecificBehaviorTest, ShardRoutingCustomTest, SharedFSAutoReplicaFailoverUtilsTest, LoggingHandlerTest, SortByFunctionTest, OutOfBoxZkACLAndCredentialsProvidersTest, SliceStateTest, TestFuzzyAnalyzedSuggestions, TestSchemaNameResource, BufferStoreTest, MoreLikeThisHandlerTest, TestClusterProperties, TestDeleteCollectionOnDownNodes, TestSegmentSorting, TestSystemIdResolver, ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, OverseerTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, TestReplicationHandler, DeleteNodeTest, DeleteShardTest, DistribJoinFromCollectionTest, LeaderFailureAfterFreshStartTest, MultiThreadedOCPTest, RollingRestartTest, TestOnReconnectListenerSupport, TestRebalanceLeaders, TestRequestForwarding, TestSizeLimitedDistributedMap, TestStressCloudBlindAtomicUpdates, TestTolerantUpdateProcessorRandomCloud, TriLevelCompositeIdRoutingTest, HdfsCollectionsAPIDistributedZkTest, HdfsRecoverLeaseTest, HdfsSyncSliceTest, HdfsUnloadDistributedZkTest]
[junit4] Completed [548/711 (1!)] on J1 in 70.73s, 1 test, 1 error <<< FAILURES!
[...truncated 43964 lines...]