You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2020/08/01 09:10:42 UTC

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 2271 - Still Unstable

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/2271/

1 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore

Error Message:
ObjectTracker found 6 object(s) that were not released!!! [MMapDirectory, InternalHttpClient, MMapDirectory, SolrCore, MMapDirectory, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:523)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:354)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:425)  at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1216)  at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)  at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)  at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)  at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)  at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348)  at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)  at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)  at org.apache.solr.handler.ReplicationHandler.inform(ReplicationHandler.java:1247)  at org.apache.solr.cloud.ReplicateFromLeader.startReplication(ReplicateFromLeader.java:109)  at org.apache.solr.cloud.ZkController.startReplicationFromLeader(ZkController.java:1327)  at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:414)  at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:336)  at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)  at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)  at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218)  at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)  at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:558)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1002)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.server.Server.handle(Server.java:500)  at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)  at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)  at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1097)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.server.Server.handle(Server.java:500)  at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)  at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)  at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:523)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:354)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:425)  at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:252)  at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:394)  at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:336)  at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)  at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)  at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218)  at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)  at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348)  at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)  at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:421)  at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:252)  at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:394)  at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:336)  at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)  at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)  at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218)  at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)  at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)  at java.base/java.lang.Thread.run(Thread.java:834)   expected null, but was:<ObjectTracker found 6 object(s) that were not released!!! [MMapDirectory, InternalHttpClient, MMapDirectory, SolrCore, MMapDirectory, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:523)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:354)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:425)  at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1216)  at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)  at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)  at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)  at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)  at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348)  at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)  at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)  at org.apache.solr.handler.ReplicationHandler.inform(ReplicationHandler.java:1247)  at org.apache.solr.cloud.ReplicateFromLeader.startReplication(ReplicateFromLeader.java:109)  at org.apache.solr.cloud.ZkController.startReplicationFromLeader(ZkController.java:1327)  at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:414)  at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:336)  at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)  at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)  at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218)  at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)  at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:558)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1002)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.server.Server.handle(Server.java:500)  at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)  at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)  at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1097)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.server.Server.handle(Server.java:500)  at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)  at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)  at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:523)  at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:354)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:425)  at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:252)  at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:394)  at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:336)  at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)  at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)  at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218)  at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)  at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348)  at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)  at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)  at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:421)  at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:252)  at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:394)  at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:336)  at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)  at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)  at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218)  at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)  at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)  at java.base/java.lang.Thread.run(Thread.java:834)  >

Stack Trace:
java.lang.AssertionError: ObjectTracker found 6 object(s) that were not released!!! [MMapDirectory, InternalHttpClient, MMapDirectory, SolrCore, MMapDirectory, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:523)
	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:354)
	at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:425)
	at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1216)
	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
	at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)
	at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348)
	at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)
	at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)
	at org.apache.solr.handler.ReplicationHandler.inform(ReplicationHandler.java:1247)
	at org.apache.solr.cloud.ReplicateFromLeader.startReplication(ReplicateFromLeader.java:109)
	at org.apache.solr.cloud.ZkController.startReplicationFromLeader(ZkController.java:1327)
	at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:414)
	at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:336)
	at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317)
	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:558)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1002)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:500)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1097)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:500)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:523)
	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:354)
	at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:425)
	at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:252)
	at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:394)
	at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:336)
	at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317)
	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348)
	at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)
	at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)
	at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:421)
	at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:252)
	at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:394)
	at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:336)
	at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317)
	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
	at java.base/java.lang.Thread.run(Thread.java:834)

 expected null, but was:<ObjectTracker found 6 object(s) that were not released!!! [MMapDirectory, InternalHttpClient, MMapDirectory, SolrCore, MMapDirectory, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:523)
	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:354)
	at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:425)
	at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1216)
	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
	at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)
	at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348)
	at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)
	at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)
	at org.apache.solr.handler.ReplicationHandler.inform(ReplicationHandler.java:1247)
	at org.apache.solr.cloud.ReplicateFromLeader.startReplication(ReplicateFromLeader.java:109)
	at org.apache.solr.cloud.ZkController.startReplicationFromLeader(ZkController.java:1327)
	at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:414)
	at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:336)
	at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317)
	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:558)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1002)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:500)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1097)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:500)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:523)
	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:354)
	at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:425)
	at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:252)
	at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:394)
	at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:336)
	at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317)
	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348)
	at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)
	at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)
	at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:421)
	at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:252)
	at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:394)
	at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:336)
	at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317)
	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
	at java.base/java.lang.Thread.run(Thread.java:834)

>
	at __randomizedtesting.SeedInfo.seed([D6EA84D1E35130F6]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.failNotNull(Assert.java:755)
	at org.junit.Assert.assertNull(Assert.java:737)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:328)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:905)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)




Build Log:
[...truncated 13512 lines...]
   [junit4] Suite: org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore
   [junit4]   2> 3319519 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to test-framework derived value of '/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/server/solr/configsets/_default/conf'
   [junit4]   2> 3319519 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_D6EA84D1E35130F6-001/data-dir-229-001
   [junit4]   2> 3319520 WARN  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=28 numCloses=28
   [junit4]   2> 3319520 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 3319520 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, clientAuth=0.0/0.0)
   [junit4]   2> 3319520 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 3319595 WARN  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 3319597 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.6+8-LTS
   [junit4]   2> 3319603 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3319603 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3319603 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 3319606 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@11a0ead4{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 3319730 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@4e0f5c84{hdfs,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/jetty-localhost_localdomain-40617-hadoop-hdfs-3_2_0-tests_jar-_-any-14865454695165664907.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/hdfs}
   [junit4]   2> 3319730 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.e.j.s.AbstractConnector Started ServerConnector@13a37c{HTTP/1.1, (http/1.1)}{localhost.localdomain:40617}
   [junit4]   2> 3319730 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.e.j.s.Server Started @3319761ms
   [junit4]   2> 3319811 WARN  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 3319812 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.6+8-LTS
   [junit4]   2> 3319813 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3319813 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3319813 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 3319813 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@635e52ba{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 3319936 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@56fbc954{datanode,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/jetty-localhost-34807-hadoop-hdfs-3_2_0-tests_jar-_-any-15198870743294925418.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/datanode}
   [junit4]   2> 3319936 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.e.j.s.AbstractConnector Started ServerConnector@2f46499d{HTTP/1.1, (http/1.1)}{localhost:34807}
   [junit4]   2> 3319936 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.e.j.s.Server Started @3319967ms
   [junit4]   2> 3320101 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0x18639185e3e2b069: Processing first storage report for DS-b6875b07-8e52-4753-868a-b16a0e8032d3 from datanode 36f0bca3-26e7-4e77-ada1-dbaf456fd9ca
   [junit4]   2> 3320101 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0x18639185e3e2b069: from storage DS-b6875b07-8e52-4753-868a-b16a0e8032d3 node DatanodeRegistration(127.0.0.1:42179, datanodeUuid=36f0bca3-26e7-4e77-ada1-dbaf456fd9ca, infoPort=40157, infoSecurePort=0, ipcPort=38369, storageInfo=lv=-57;cid=testClusterID;nsid=1413018308;c=1596269352819), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 3320101 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0x18639185e3e2b069: Processing first storage report for DS-3720323a-a433-4933-baaa-b67e771443a9 from datanode 36f0bca3-26e7-4e77-ada1-dbaf456fd9ca
   [junit4]   2> 3320101 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0x18639185e3e2b069: from storage DS-3720323a-a433-4933-baaa-b67e771443a9 node DatanodeRegistration(127.0.0.1:42179, datanodeUuid=36f0bca3-26e7-4e77-ada1-dbaf456fd9ca, infoPort=40157, infoSecurePort=0, ipcPort=38369, storageInfo=lv=-57;cid=testClusterID;nsid=1413018308;c=1596269352819), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 3320162 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_D6EA84D1E35130F6-001/tempDir-002
   [junit4]   2> 3320163 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 3320165 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port: 0.0.0.0/0.0.0.0:0
   [junit4]   2> 3320165 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 3320167 WARN  (ZkTestServer Run Thread) [     ] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
   [junit4]   2> 3320265 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.c.ZkTestServer start zk server on port: 33829
   [junit4]   2> 3320265 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:33829
   [junit4]   2> 3320266 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:33829
   [junit4]   2> 3320266 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 33829
   [junit4]   2> 3320306 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3320345 INFO  (zkConnectionManagerCallback-26519-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3320345 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3320433 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3320457 INFO  (zkConnectionManagerCallback-26521-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3320457 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3320481 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3320497 INFO  (zkConnectionManagerCallback-26523-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3320497 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3320499 WARN  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker-SendThread(127.0.0.1:33829)) [     ] o.a.z.ClientCnxn An exception was thrown while closing send thread for session 0x100a1f01d2c0002.
   [junit4]   2>           => EndOfStreamException: Unable to read additional data from server sessionid 0x100a1f01d2c0002, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:75)
   [junit4]   2> org.apache.zookeeper.ClientCnxn$EndOfStreamException: Unable to read additional data from server sessionid 0x100a1f01d2c0002, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:75) ~[zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:348) ~[zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1262) [zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 3320608 INFO  (jetty-launcher-26524-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 3320608 INFO  (jetty-launcher-26524-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
   [junit4]   2> 3320608 INFO  (jetty-launcher-26524-thread-1) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.6+8-LTS
   [junit4]   2> 3320626 INFO  (jetty-launcher-26524-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 3320626 INFO  (jetty-launcher-26524-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
   [junit4]   2> 3320626 INFO  (jetty-launcher-26524-thread-2) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.6+8-LTS
   [junit4]   2> 3320643 INFO  (jetty-launcher-26524-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3320643 INFO  (jetty-launcher-26524-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3320643 INFO  (jetty-launcher-26524-thread-1) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 3320644 INFO  (jetty-launcher-26524-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3320644 INFO  (jetty-launcher-26524-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3320644 INFO  (jetty-launcher-26524-thread-2) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 3320644 INFO  (jetty-launcher-26524-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@61c57740{/solr,null,AVAILABLE}
   [junit4]   2> 3320644 INFO  (jetty-launcher-26524-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@718ed3a7{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:43039}
   [junit4]   2> 3320644 INFO  (jetty-launcher-26524-thread-2) [     ] o.e.j.s.Server Started @3320675ms
   [junit4]   2> 3320644 INFO  (jetty-launcher-26524-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43039}
   [junit4]   2> 3320644 ERROR (jetty-launcher-26524-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3320644 INFO  (jetty-launcher-26524-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3320645 INFO  (jetty-launcher-26524-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 9.0.0
   [junit4]   2> 3320645 INFO  (jetty-launcher-26524-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3320645 INFO  (jetty-launcher-26524-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
   [junit4]   2> 3320645 INFO  (jetty-launcher-26524-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-08-01T08:09:13.927086Z
   [junit4]   2> 3320645 INFO  (jetty-launcher-26524-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@78932ee8{/solr,null,AVAILABLE}
   [junit4]   2> 3320649 INFO  (jetty-launcher-26524-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@2b224a4{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:36425}
   [junit4]   2> 3320649 INFO  (jetty-launcher-26524-thread-1) [     ] o.e.j.s.Server Started @3320680ms
   [junit4]   2> 3320649 INFO  (jetty-launcher-26524-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=36425}
   [junit4]   2> 3320650 ERROR (jetty-launcher-26524-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3320650 INFO  (jetty-launcher-26524-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3320650 INFO  (jetty-launcher-26524-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 9.0.0
   [junit4]   2> 3320650 INFO  (jetty-launcher-26524-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3320650 INFO  (jetty-launcher-26524-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
   [junit4]   2> 3320650 INFO  (jetty-launcher-26524-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-08-01T08:09:13.932126Z
   [junit4]   2> 3320665 INFO  (jetty-launcher-26524-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3320674 INFO  (jetty-launcher-26524-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3320675 INFO  (zkConnectionManagerCallback-26526-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3320676 INFO  (jetty-launcher-26524-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3320676 INFO  (jetty-launcher-26524-thread-2) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3320676 INFO  (zkConnectionManagerCallback-26528-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3320676 INFO  (jetty-launcher-26524-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3320678 INFO  (jetty-launcher-26524-thread-1) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3320680 INFO  (jetty-launcher-26524-thread-2) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3ad262c6, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 3320708 INFO  (jetty-launcher-26524-thread-1) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3ad262c6, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 3320709 WARN  (jetty-launcher-26524-thread-2-SendThread(127.0.0.1:33829)) [     ] o.a.z.ClientCnxn An exception was thrown while closing send thread for session 0x100a1f01d2c0003.
   [junit4]   2>           => EndOfStreamException: Unable to read additional data from server sessionid 0x100a1f01d2c0003, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:75)
   [junit4]   2> org.apache.zookeeper.ClientCnxn$EndOfStreamException: Unable to read additional data from server sessionid 0x100a1f01d2c0003, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:75) ~[zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:348) ~[zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1262) [zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 3321731 INFO  (jetty-launcher-26524-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 3321731 WARN  (jetty-launcher-26524-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@197a2e61[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3321731 WARN  (jetty-launcher-26524-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@197a2e61[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3321733 WARN  (jetty-launcher-26524-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@735b74a2[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3321733 WARN  (jetty-launcher-26524-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@735b74a2[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3321735 INFO  (jetty-launcher-26524-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33829/solr
   [junit4]   2> 3321738 INFO  (jetty-launcher-26524-thread-1) [n:127.0.0.1:36425_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3321754 INFO  (zkConnectionManagerCallback-26541-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3321754 INFO  (jetty-launcher-26524-thread-1) [n:127.0.0.1:36425_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3321856 INFO  (jetty-launcher-26524-thread-1) [n:127.0.0.1:36425_solr     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:36425_solr
   [junit4]   2> 3321857 INFO  (jetty-launcher-26524-thread-1) [n:127.0.0.1:36425_solr     ] o.a.s.c.Overseer Overseer (id=72235646691573765-127.0.0.1:36425_solr-n_0000000000) starting
   [junit4]   2> 3321863 INFO  (OverseerStateUpdate-72235646691573765-127.0.0.1:36425_solr-n_0000000000) [n:127.0.0.1:36425_solr     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:36425_solr
   [junit4]   2> 3321870 INFO  (jetty-launcher-26524-thread-1) [n:127.0.0.1:36425_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36425_solr
   [junit4]   2> 3321871 INFO  (jetty-launcher-26524-thread-1) [n:127.0.0.1:36425_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 3321872 WARN  (jetty-launcher-26524-thread-1) [n:127.0.0.1:36425_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 3321872 INFO  (jetty-launcher-26524-thread-1) [n:127.0.0.1:36425_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = hdfs,class = org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = {name=hdfs, class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = {location=/backup, solr.hdfs.home=hdfs://localhost.localdomain:40893/solr, solr.hdfs.confdir=}}
   [junit4]   2> 3321872 INFO  (jetty-launcher-26524-thread-1) [n:127.0.0.1:36425_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = poisioned,class = org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository,attributes = {default=true, name=poisioned, class=org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository},}
   [junit4]   2> 3321872 INFO  (jetty-launcher-26524-thread-1) [n:127.0.0.1:36425_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Default configuration for backup repository is with configuration params {type = repository,name = poisioned,class = org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository,attributes = {default=true, name=poisioned, class=org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository},}
   [junit4]   2> 3321886 INFO  (zkCallback-26540-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3321897 INFO  (jetty-launcher-26524-thread-1) [n:127.0.0.1:36425_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 3321914 INFO  (jetty-launcher-26524-thread-1) [n:127.0.0.1:36425_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3ad262c6
   [junit4]   2> 3321921 INFO  (jetty-launcher-26524-thread-1) [n:127.0.0.1:36425_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3ad262c6
   [junit4]   2> 3321921 INFO  (jetty-launcher-26524-thread-1) [n:127.0.0.1:36425_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3ad262c6
   [junit4]   2> 3321922 INFO  (jetty-launcher-26524-thread-1) [n:127.0.0.1:36425_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_D6EA84D1E35130F6-001/tempDir-002/node1
   [junit4]   2> 3322006 INFO  (jetty-launcher-26524-thread-2) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 3322007 WARN  (jetty-launcher-26524-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@1f765b80[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3322007 WARN  (jetty-launcher-26524-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@1f765b80[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3322010 WARN  (jetty-launcher-26524-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2c8c717f[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3322011 WARN  (jetty-launcher-26524-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2c8c717f[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3322012 INFO  (jetty-launcher-26524-thread-2) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33829/solr
   [junit4]   2> 3322013 INFO  (jetty-launcher-26524-thread-2) [n:127.0.0.1:43039_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3322014 INFO  (zkConnectionManagerCallback-26558-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3322014 INFO  (jetty-launcher-26524-thread-2) [n:127.0.0.1:43039_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3322018 INFO  (jetty-launcher-26524-thread-2) [n:127.0.0.1:43039_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3322022 INFO  (jetty-launcher-26524-thread-2) [n:127.0.0.1:43039_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:43039_solr as DOWN
   [junit4]   2> 3322023 INFO  (jetty-launcher-26524-thread-2) [n:127.0.0.1:43039_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 1000 transient cores
   [junit4]   2> 3322023 INFO  (jetty-launcher-26524-thread-2) [n:127.0.0.1:43039_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43039_solr
   [junit4]   2> 3322023 INFO  (zkCallback-26540-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3322025 INFO  (jetty-launcher-26524-thread-2) [n:127.0.0.1:43039_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 3322025 WARN  (jetty-launcher-26524-thread-2) [n:127.0.0.1:43039_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 3322025 INFO  (jetty-launcher-26524-thread-2) [n:127.0.0.1:43039_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = hdfs,class = org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = {name=hdfs, class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = {location=/backup, solr.hdfs.home=hdfs://localhost.localdomain:40893/solr, solr.hdfs.confdir=}}
   [junit4]   2> 3322025 INFO  (jetty-launcher-26524-thread-2) [n:127.0.0.1:43039_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = poisioned,class = org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository,attributes = {default=true, name=poisioned, class=org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository},}
   [junit4]   2> 3322025 INFO  (jetty-launcher-26524-thread-2) [n:127.0.0.1:43039_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Default configuration for backup repository is with configuration params {type = repository,name = poisioned,class = org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository,attributes = {default=true, name=poisioned, class=org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository},}
   [junit4]   2> 3322026 INFO  (zkCallback-26557-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3322046 INFO  (jetty-launcher-26524-thread-2) [n:127.0.0.1:43039_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 3322068 INFO  (jetty-launcher-26524-thread-2) [n:127.0.0.1:43039_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3ad262c6
   [junit4]   2> 3322078 INFO  (jetty-launcher-26524-thread-2) [n:127.0.0.1:43039_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3ad262c6
   [junit4]   2> 3322078 INFO  (jetty-launcher-26524-thread-2) [n:127.0.0.1:43039_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3ad262c6
   [junit4]   2> 3322079 INFO  (jetty-launcher-26524-thread-2) [n:127.0.0.1:43039_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_D6EA84D1E35130F6-001/tempDir-002/node2
   [junit4]   2> 3322088 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=2
   [junit4]   2> 3322089 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 3322090 INFO  (zkConnectionManagerCallback-26571-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3322090 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 3322091 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3322092 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[D6EA84D1E35130F6]-worker) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33829/solr ready
   [junit4]   2> 3322122 INFO  (TEST-TestHdfsCloudBackupRestore.testRestoreFailure-seed#[D6EA84D1E35130F6]) [     ] o.a.s.SolrTestCaseJ4 ###Starting testRestoreFailure
   [junit4]   2> 3322124 INFO  (qtp209046362-66575) [n:127.0.0.1:36425_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=1&collection.configName=conf1&name=hdfsbackuprestore_testfailure&nrtReplicas=2&action=CREATE&numShards=2&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3322127 INFO  (OverseerThreadFactory-26546-thread-1-processing-n:127.0.0.1:36425_solr) [n:127.0.0.1:36425_solr     ] o.a.s.c.a.c.CreateCollectionCmd Create collection hdfsbackuprestore_testfailure
   [junit4]   2> 3322231 WARN  (OverseerThreadFactory-26546-thread-1-processing-n:127.0.0.1:36425_solr) [n:127.0.0.1:36425_solr     ] o.a.s.c.a.c.CreateCollectionCmd Specified number of replicas of 3 on collection hdfsbackuprestore_testfailure is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
   [junit4]   2> 3322234 INFO  (OverseerStateUpdate-72235646691573765-127.0.0.1:36425_solr-n_0000000000) [n:127.0.0.1:36425_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"hdfsbackuprestore_testfailure",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"hdfsbackuprestore_testfailure_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:43039/solr",
   [junit4]   2>   "node_name":"127.0.0.1:43039_solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 3322237 INFO  (OverseerStateUpdate-72235646691573765-127.0.0.1:36425_solr-n_0000000000) [n:127.0.0.1:36425_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"hdfsbackuprestore_testfailure",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"hdfsbackuprestore_testfailure_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:36425/solr",
   [junit4]   2>   "node_name":"127.0.0.1:36425_solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 3322239 INFO  (OverseerStateUpdate-72235646691573765-127.0.0.1:36425_solr-n_0000000000) [n:127.0.0.1:36425_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"hdfsbackuprestore_testfailure",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"hdfsbackuprestore_testfailure_shard1_replica_p4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:43039/solr",
   [junit4]   2>   "node_name":"127.0.0.1:43039_solr",
   [junit4]   2>   "type":"PULL",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 3322241 INFO  (OverseerStateUpdate-72235646691573765-127.0.0.1:36425_solr-n_0000000000) [n:1

[...truncated too long message...]

ScopedHandler.nextScope(ScopedHandler.java:186)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]    > 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
   [junit4]    > 	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
   [junit4]    > 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
   [junit4]    > 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
   [junit4]    > 	at org.eclipse.jetty.server.Server.handle(Server.java:500)
   [junit4]    > 	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
   [junit4]    > 	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
   [junit4]    > 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
   [junit4]    > 	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
   [junit4]    > 	at java.base/java.lang.Thread.run(Thread.java:834)
   [junit4]    > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
   [junit4]    > 	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
   [junit4]    > 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1097)
   [junit4]    > 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)
   [junit4]    > 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
   [junit4]    > 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
   [junit4]    > 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
   [junit4]    > 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
   [junit4]    > 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)
   [junit4]    > 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)
   [junit4]    > 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)
   [junit4]    > 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
   [junit4]    > 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]    > 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
   [junit4]    > 	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
   [junit4]    > 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
   [junit4]    > 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
   [junit4]    > 	at org.eclipse.jetty.server.Server.handle(Server.java:500)
   [junit4]    > 	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
   [junit4]    > 	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
   [junit4]    > 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
   [junit4]    > 	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
   [junit4]    > 	at java.base/java.lang.Thread.run(Thread.java:834)
   [junit4]    > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
   [junit4]    > 	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
   [junit4]    > 	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
   [junit4]    > 	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:523)
   [junit4]    > 	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:354)
   [junit4]    > 	at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:425)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:252)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:394)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:336)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317)
   [junit4]    > 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
   [junit4]    > 	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
   [junit4]    > 	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218)
   [junit4]    > 	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
   [junit4]    > 	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
   [junit4]    > 	at java.base/java.lang.Thread.run(Thread.java:834)
   [junit4]    > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
   [junit4]    > 	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348)
   [junit4]    > 	at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)
   [junit4]    > 	at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)
   [junit4]    > 	at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:421)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:252)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:394)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:336)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317)
   [junit4]    > 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
   [junit4]    > 	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
   [junit4]    > 	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:218)
   [junit4]    > 	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
   [junit4]    > 	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
   [junit4]    > 	at java.base/java.lang.Thread.run(Thread.java:834)
   [junit4]    > >
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D6EA84D1E35130F6]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:328)
   [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
   [junit4]    > 	at java.base/java.lang.Thread.run(Thread.java:834)
   [junit4] Completed [394/868 (1!)] on J1 in 88.58s, 2 tests, 1 failure <<< FAILURES!

[...truncated 53377 lines...]

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 2272 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/2272/

3 tests failed.
FAILED:  org.apache.solr.cloud.hdfs.StressHdfsTest.test

Error Message:
Error from server at http://127.0.0.1:36235/delete_data_dir: Expected mime type application/octet-stream but got application/json. {   "error":{     "metadata":[       "error-class","org.apache.solr.common.SolrException",       "root-error-class","java.net.SocketTimeoutException"],     "msg":"Error trying to proxy request for url: http://127.0.0.1:38283/delete_data_dir/update",     "trace":"org.apache.solr.common.SolrException: Error trying to proxy request for url: http://127.0.0.1:38283/delete_data_dir/update\n\tat org.apache.solr.servlet.HttpSolrCall.remoteQuery(HttpSolrCall.java:747)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:571)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:500)\n\tat org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)\n\tat org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:273)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)\n\tat org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)\n\tat java.base/java.lang.Thread.run(Thread.java:834)\nCaused by: java.net.SocketTimeoutException: Read timed out\n\tat java.base/java.net.SocketInputStream.socketRead0(Native Method)\n\tat java.base/java.net.SocketInputStream.socketRead(SocketInputStream.java:115)\n\tat java.base/java.net.SocketInputStream.read(SocketInputStream.java:168)\n\tat java.base/java.net.SocketInputStream.read(SocketInputStream.java:140)\n\tat org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)\n\tat org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)\n\tat org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)\n\tat org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)\n\tat org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)\n\tat org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)\n\tat org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)\n\tat org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)\n\tat org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)\n\tat org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)\n\tat org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:122)\n\tat org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)\n\tat org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)\n\tat org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)\n\tat org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)\n\tat org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)\n\tat org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)\n\tat org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)\n\tat org.apache.solr.servlet.HttpSolrCall.remoteQuery(HttpSolrCall.java:719)\n\t... 32 more\n",     "code":500}} 

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:36235/delete_data_dir: Expected mime type application/octet-stream but got application/json. {
  "error":{
    "metadata":[
      "error-class","org.apache.solr.common.SolrException",
      "root-error-class","java.net.SocketTimeoutException"],
    "msg":"Error trying to proxy request for url: http://127.0.0.1:38283/delete_data_dir/update",
    "trace":"org.apache.solr.common.SolrException: Error trying to proxy request for url: http://127.0.0.1:38283/delete_data_dir/update\n\tat org.apache.solr.servlet.HttpSolrCall.remoteQuery(HttpSolrCall.java:747)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:571)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:500)\n\tat org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)\n\tat org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:273)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)\n\tat org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)\n\tat java.base/java.lang.Thread.run(Thread.java:834)\nCaused by: java.net.SocketTimeoutException: Read timed out\n\tat java.base/java.net.SocketInputStream.socketRead0(Native Method)\n\tat java.base/java.net.SocketInputStream.socketRead(SocketInputStream.java:115)\n\tat java.base/java.net.SocketInputStream.read(SocketInputStream.java:168)\n\tat java.base/java.net.SocketInputStream.read(SocketInputStream.java:140)\n\tat org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)\n\tat org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)\n\tat org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)\n\tat org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)\n\tat org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)\n\tat org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)\n\tat org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)\n\tat org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)\n\tat org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)\n\tat org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)\n\tat org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:122)\n\tat org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)\n\tat org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)\n\tat org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)\n\tat org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)\n\tat org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)\n\tat org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)\n\tat org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)\n\tat org.apache.solr.servlet.HttpSolrCall.remoteQuery(HttpSolrCall.java:719)\n\t... 32 more\n",
    "code":500}}

	at __randomizedtesting.SeedInfo.seed([97773E4948CA580B:1F230193E63635F3]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:635)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:210)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:505)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:480)
	at org.apache.solr.cloud.hdfs.StressHdfsTest.createAndDeleteCollection(StressHdfsTest.java:190)
	at org.apache.solr.cloud.hdfs.StressHdfsTest.test(StressHdfsTest.java:107)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestSolrConfigHandlerConcurrent

Error Message:
ObjectTracker found 3 object(s) that were not released!!! [MockDirectoryWrapper, RawDirectoryWrapper, MockDirectoryWrapper] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:558)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1002)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.server.Server.handle(Server.java:500)  at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)  at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)  at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:818)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1010)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.server.Server.handle(Server.java:500)  at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)  at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)  at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:407)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:786)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1010)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.server.Server.handle(Server.java:500)  at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)  at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)  at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)  at java.base/java.lang.Thread.run(Thread.java:834)   expected null, but was:<ObjectTracker found 3 object(s) that were not released!!! [MockDirectoryWrapper, RawDirectoryWrapper, MockDirectoryWrapper] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:558)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1002)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.server.Server.handle(Server.java:500)  at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)  at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)  at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:818)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1010)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.server.Server.handle(Server.java:500)  at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)  at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)  at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:407)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:786)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1010)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.server.Server.handle(Server.java:500)  at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)  at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)  at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)  at java.base/java.lang.Thread.run(Thread.java:834)  >

Stack Trace:
java.lang.AssertionError: ObjectTracker found 3 object(s) that were not released!!! [MockDirectoryWrapper, RawDirectoryWrapper, MockDirectoryWrapper]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:558)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1002)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:500)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)
	at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:818)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1010)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:500)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:407)
	at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:786)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1010)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:500)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
	at java.base/java.lang.Thread.run(Thread.java:834)

 expected null, but was:<ObjectTracker found 3 object(s) that were not released!!! [MockDirectoryWrapper, RawDirectoryWrapper, MockDirectoryWrapper]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:558)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1002)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:500)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)
	at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:818)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1010)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:500)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:407)
	at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:786)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1010)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:926)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:500)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
	at java.base/java.lang.Thread.run(Thread.java:834)

>
	at __randomizedtesting.SeedInfo.seed([97773E4948CA580B]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.failNotNull(Assert.java:755)
	at org.junit.Assert.assertNull(Assert.java:737)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:328)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:905)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.client.solrj.io.stream.MathExpressionTest.testZipFDistribution

Error Message:
Zipf distribution not descending!!!

Stack Trace:
java.lang.Exception: Zipf distribution not descending!!!
	at __randomizedtesting.SeedInfo.seed([2155D61EB11F1623:5E0BB3AA6B71E0B]:0)
	at org.apache.solr.client.solrj.io.stream.MathExpressionTest.testZipFDistribution(MathExpressionTest.java:3871)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)




Build Log:
[...truncated 13939 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.StressHdfsTest
   [junit4]   2> 4207905 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to test-framework derived value of '/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/server/solr/configsets/_default/conf'
   [junit4]   2> 4207905 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 4207905 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.StressHdfsTest_97773E4948CA580B-001/data-dir-214-001
   [junit4]   2> 4207906 WARN  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=336 numCloses=336
   [junit4]   2> 4207906 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 4207907 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl="https://issues.apache.org/jira/browse/SOLR-5776")
   [junit4]   2> 4207907 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 4207993 WARN  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 4207994 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.6+8-LTS
   [junit4]   2> 4207996 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4207996 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4207996 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 4207996 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5cbb8379{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 4208119 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@909d167{hdfs,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/jetty-localhost_localdomain-38733-hadoop-hdfs-3_2_0-tests_jar-_-any-3290341024676787156.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/hdfs}
   [junit4]   2> 4208119 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.e.j.s.AbstractConnector Started ServerConnector@3b442e67{HTTP/1.1, (http/1.1)}{localhost.localdomain:38733}
   [junit4]   2> 4208119 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.e.j.s.Server Started @4208152ms
   [junit4]   2> 4208184 WARN  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 4208185 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.6+8-LTS
   [junit4]   2> 4208185 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4208185 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4208185 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 4208193 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6d62f214{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 4208336 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@3ece7986{datanode,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/jetty-localhost-41313-hadoop-hdfs-3_2_0-tests_jar-_-any-3646037582428078022.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/datanode}
   [junit4]   2> 4208336 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.e.j.s.AbstractConnector Started ServerConnector@7df7afd6{HTTP/1.1, (http/1.1)}{localhost:41313}
   [junit4]   2> 4208336 INFO  (SUITE-StressHdfsTest-seed#[97773E4948CA580B]-worker) [     ] o.e.j.s.Server Started @4208369ms
   [junit4]   2> 4208508 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0xe6ac84cf24a878fa: Processing first storage report for DS-0b096ad0-e781-444f-b4a7-22eebb8c69a2 from datanode 1543e145-4adf-40d3-9b64-10d22232add6
   [junit4]   2> 4208508 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0xe6ac84cf24a878fa: from storage DS-0b096ad0-e781-444f-b4a7-22eebb8c69a2 node DatanodeRegistration(127.0.0.1:33689, datanodeUuid=1543e145-4adf-40d3-9b64-10d22232add6, infoPort=39411, infoSecurePort=0, ipcPort=35037, storageInfo=lv=-57;cid=testClusterID;nsid=662182859;c=1596360393426), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 4208508 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0xe6ac84cf24a878fa: Processing first storage report for DS-492f7013-afa3-4e9e-9dd5-c1d3e8b97049 from datanode 1543e145-4adf-40d3-9b64-10d22232add6
   [junit4]   2> 4208508 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0xe6ac84cf24a878fa: from storage DS-492f7013-afa3-4e9e-9dd5-c1d3e8b97049 node DatanodeRegistration(127.0.0.1:33689, datanodeUuid=1543e145-4adf-40d3-9b64-10d22232add6, infoPort=39411, infoSecurePort=0, ipcPort=35037, storageInfo=lv=-57;cid=testClusterID;nsid=662182859;c=1596360393426), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 4208582 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 4208582 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port: 0.0.0.0/0.0.0.0:0
   [junit4]   2> 4208582 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 4208584 WARN  (ZkTestServer Run Thread) [     ] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
   [junit4]   2> 4208682 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.ZkTestServer start zk server on port: 38767
   [junit4]   2> 4208682 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:38767
   [junit4]   2> 4208682 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:38767
   [junit4]   2> 4208682 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 38767
   [junit4]   2> 4208684 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4208690 INFO  (zkConnectionManagerCallback-32366-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4208690 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4208699 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4208701 INFO  (zkConnectionManagerCallback-32368-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4208701 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4208702 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 4208703 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 4208705 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 4208706 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 4208708 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 4208709 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 4208711 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 4208712 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 4208713 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 4208715 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 4208716 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 4208727 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 4208952 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 4208953 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
   [junit4]   2> 4208953 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.6+8-LTS
   [junit4]   2> 4208964 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4208964 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4208964 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 4208964 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3882f9b{/,null,AVAILABLE}
   [junit4]   2> 4208964 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.e.j.s.AbstractConnector Started ServerConnector@3bf9c78c{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:41053}
   [junit4]   2> 4208964 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.e.j.s.Server Started @4208997ms
   [junit4]   2> 4208964 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/, solr.data.dir=hdfs://localhost.localdomain:38827/hdfs__localhost.localdomain_38827__home_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.StressHdfsTest_97773E4948CA580B-001_tempDir-002_control_data, hostPort=41053, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.StressHdfsTest_97773E4948CA580B-001/control-001/cores}
   [junit4]   2> 4208965 ERROR (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4208965 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4208965 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 9.0.0
   [junit4]   2> 4208965 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4208965 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr
   [junit4]   2> 4208965 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-08-02T09:26:34.469261Z
   [junit4]   2> 4208965 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4208967 INFO  (zkConnectionManagerCallback-32370-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4208967 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4209068 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4209068 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.StressHdfsTest_97773E4948CA580B-001/control-001/solr.xml
   [junit4]   2> 4209073 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@186f01d2, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4209982 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 4209983 WARN  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@289c8b3a[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4209983 WARN  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@289c8b3a[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4209985 WARN  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@1215c174[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4209985 WARN  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@1215c174[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4209987 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38767/solr
   [junit4]   2> 4209988 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4209990 INFO  (zkConnectionManagerCallback-32380-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4209990 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4210092 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [n:127.0.0.1:41053_     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4210093 INFO  (zkConnectionManagerCallback-32382-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4210093 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [n:127.0.0.1:41053_     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4210202 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [n:127.0.0.1:41053_     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:41053_
   [junit4]   2> 4210203 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [n:127.0.0.1:41053_     ] o.a.s.c.Overseer Overseer (id=72241613130563588-127.0.0.1:41053_-n_0000000000) starting
   [junit4]   2> 4210208 INFO  (OverseerStateUpdate-72241613130563588-127.0.0.1:41053_-n_0000000000) [n:127.0.0.1:41053_     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:41053_
   [junit4]   2> 4210208 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [n:127.0.0.1:41053_     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41053_
   [junit4]   2> 4210209 INFO  (OverseerStateUpdate-72241613130563588-127.0.0.1:41053_-n_0000000000) [n:127.0.0.1:41053_     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4210211 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [n:127.0.0.1:41053_     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 4210212 WARN  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [n:127.0.0.1:41053_     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 4210239 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [n:127.0.0.1:41053_     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 4210259 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [n:127.0.0.1:41053_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@186f01d2
   [junit4]   2> 4210268 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [n:127.0.0.1:41053_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@186f01d2
   [junit4]   2> 4210268 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [n:127.0.0.1:41053_     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@186f01d2
   [junit4]   2> 4210269 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [n:127.0.0.1:41053_     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.StressHdfsTest_97773E4948CA580B-001/control-001/cores
   [junit4]   2> 4210276 INFO  (TEST-StressHdfsTest.test-seed#[97773E4948CA580B]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4210277 INFO  (zkConnectionManagerCallback-32397-thread-

[...truncated too long message...]

]   2> 221866 INFO  (jetty-closer-4264-thread-4) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty tag=null
   [junit4]   2> 221866 INFO  (jetty-closer-4264-thread-4) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7e9be7a9: rootName = solr_40439, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty/com.codahale.metrics.MetricRegistry@1323a02
   [junit4]   2> 221866 INFO  (jetty-closer-4264-thread-3) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm tag=null
   [junit4]   2> 221866 INFO  (jetty-closer-4264-thread-3) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4e0f3556: rootName = solr_33537, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm/com.codahale.metrics.MetricRegistry@1e16745c
   [junit4]   2> 221867 INFO  (jetty-closer-4264-thread-4) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster tag=null
   [junit4]   2> 221869 INFO  (jetty-closer-4264-thread-3) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty tag=null
   [junit4]   2> 221869 INFO  (jetty-closer-4264-thread-3) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@43899f90: rootName = solr_33537, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty/com.codahale.metrics.MetricRegistry@1323a02
   [junit4]   2> 221869 INFO  (jetty-closer-4264-thread-3) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster tag=null
   [junit4]   2> 221870 WARN  (qtp379954110-6572-SendThread(127.0.0.1:46635)) [     ] o.a.z.ClientCnxn An exception was thrown while closing send thread for session 0x100a77505f8000c.
   [junit4]   2>           => EndOfStreamException: Unable to read additional data from server sessionid 0x100a77505f8000c, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:75)
   [junit4]   2> org.apache.zookeeper.ClientCnxn$EndOfStreamException: Unable to read additional data from server sessionid 0x100a77505f8000c, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:75) ~[zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:348) ~[zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1262) [zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 221934 INFO  (jetty-closer-4264-thread-1) [     ] o.a.s.c.Overseer Overseer (id=72241715091079177-127.0.0.1:42073_solr-n_0000000000) closing
   [junit4]   2> 221935 INFO  (jetty-closer-4264-thread-1) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@1f52015e{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:0}
   [junit4]   2> 221935 INFO  (jetty-closer-4264-thread-1) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3a9a0685{/solr,null,UNAVAILABLE}
   [junit4]   2> 221935 INFO  (jetty-closer-4264-thread-1) [     ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 222235 INFO  (jetty-closer-4264-thread-2) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@1e2de25e{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:0}
   [junit4]   2> 222235 INFO  (jetty-closer-4264-thread-2) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@21dd5bc6{/solr,null,UNAVAILABLE}
   [junit4]   2> 222236 INFO  (jetty-closer-4264-thread-2) [     ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 222336 INFO  (jetty-closer-4264-thread-4) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@40ae368c{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:0}
   [junit4]   2> 222336 INFO  (jetty-closer-4264-thread-4) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@23d7607{/solr,null,UNAVAILABLE}
   [junit4]   2> 222336 INFO  (jetty-closer-4264-thread-4) [     ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 222436 INFO  (jetty-closer-4264-thread-3) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@595f5521{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:0}
   [junit4]   2> 222436 INFO  (jetty-closer-4264-thread-3) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@35b134ba{/solr,null,UNAVAILABLE}
   [junit4]   2> 222436 INFO  (jetty-closer-4264-thread-3) [     ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 222437 INFO  (SUITE-MathExpressionTest-seed#[2155D61EB11F1623]-worker) [     ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
   [junit4]   2> 222438 WARN  (SUITE-MathExpressionTest-seed#[2155D61EB11F1623]-worker-SendThread(127.0.0.1:46635)) [     ] o.a.z.ClientCnxn An exception was thrown while closing send thread for session 0x100a77505f80000.
   [junit4]   2>           => EndOfStreamException: Unable to read additional data from server sessionid 0x100a77505f80000, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:75)
   [junit4]   2> org.apache.zookeeper.ClientCnxn$EndOfStreamException: Unable to read additional data from server sessionid 0x100a77505f80000, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:75) ~[zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:348) ~[zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1262) [zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 222641 WARN  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	7	/solr/aliases.json
   [junit4]   2> 	4	/solr/packages.json
   [junit4]   2> 	4	/solr/security.json
   [junit4]   2> 	3	/solr/collections/collection1/terms/shard2
   [junit4]   2> 	3	/solr/collections/collection1/terms/shard1
   [junit4]   2> 	2	/solr/collections/collection1/collectionprops.json
   [junit4]   2> 	2	/solr/configs/conf
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	24	/solr/collections/collection1/state.json
   [junit4]   2> 	7	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	29	/solr/live_nodes
   [junit4]   2> 	21	/solr/overseer/queue
   [junit4]   2> 	12	/solr/collections
   [junit4]   2> 	5	/solr/overseer/collection-queue-work
   [junit4]   2> 
   [junit4]   2> 222644 INFO  (SUITE-MathExpressionTest-seed#[2155D61EB11F1623]-worker) [     ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:46635
   [junit4]   2> 222644 INFO  (SUITE-MathExpressionTest-seed#[2155D61EB11F1623]-worker) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:46635
   [junit4]   2> 222644 INFO  (SUITE-MathExpressionTest-seed#[2155D61EB11F1623]-worker) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 46635
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.MathExpressionTest_2155D61EB11F1623-001
   [junit4]   2> Aug 02, 2020 9:52:47 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene86): {test_t=PostingsFormat(name=Asserting), col_s=Lucene84, multiDefault=Lucene84, id=PostingsFormat(name=LuceneVarGapDocFreqInterval), text=PostingsFormat(name=Asserting)}, docValues:{col_s=DocValuesFormat(name=Lucene80), order_i=DocValuesFormat(name=Lucene80), _version_=DocValuesFormat(name=Lucene80), multiDefault=DocValuesFormat(name=Lucene80), price_f=DocValuesFormat(name=Lucene80), intDefault=DocValuesFormat(name=Lucene80), test_dt=DocValuesFormat(name=Asserting), price_i=DocValuesFormat(name=Asserting), id=DocValuesFormat(name=Asserting), miles_i=DocValuesFormat(name=Lucene80), timestamp=DocValuesFormat(name=Lucene80)}, maxPointsInLeafNode=354, maxMBSortInHeap=7.767988136687123, sim=Asserting(RandomSimilarity(queryNorm=false): {}), locale=fr-CI, timezone=Pacific/Kosrae
   [junit4]   2> NOTE: Linux 4.15.0-108-generic amd64/Oracle Corporation 11.0.6 (64-bit)/cpus=4,threads=1,free=202695816,total=318767104
   [junit4]   2> NOTE: All tests run in this JVM: [SolrExampleEmbeddedTest, TermsResponseTest, OperationsTest, SolrExampleStreamingBinaryTest, AndEvaluatorTest, BasicHttpSolrClientTest, LBHttpSolrClientBuilderTest, SolrExampleBinaryTest, TestValidatingJsonMap, HttpSolrClientConPoolTest, TestCollectionAdminRequest, HttpClientUtilTest, TestJSONWriter, TestRetryUtil, TestDocCollectionWatcher, TestUpdateRequestCodec, LessThanEvaluatorTest, NestableJsonFacetTest, SolrParamTest, CloudSolrClientMultiConstructorTest, StreamExpressionTest, TestLBHttp2SolrClient, ConcurrentUpdateSolrClientTest, SquareRootEvaluatorTest, GreaterThanEvaluatorTest, FieldValueEvaluatorTest, NaturalLogEvaluatorTest, TestFastWriter, JsonRequestApiTest, CollectionAdminRequestRequiredParamsTest, TestJsonRecordReader, TestDocumentObjectBinder, JettyWebappTest, LargeVolumeBinaryJettyTest, LargeVolumeEmbeddedTest, LargeVolumeJettyTest, SolrExampleStreamingTest, TestEmbeddedSolrServer, CloudHttp2SolrClientBuilderTest, CloudHttp2SolrClientMultiConstructorTest, CloudHttp2SolrClientRetryTest, CloudSolrClientBadInputTest, CloudSolrClientBuilderTest, CloudSolrClientTest, MathExpressionTest]
   [junit4] Completed [162/204 (1!)] on J1 in 17.82s, 141 tests, 1 error <<< FAILURES!

[...truncated 50899 lines...]