You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2020/05/02 21:07:37 UTC
[JENKINS] Lucene-Solr-Tests-8.x - Build # 1474 - Unstable
Build: https://builds.apache.org/job/Lucene-Solr-Tests-8.x/1474/
1 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore
Error Message:
ObjectTracker found 5 object(s) that were not released!!! [MMapDirectory, MMapDirectory, InternalHttpClient, SolrCore, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:519) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424) at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1210) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:800) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:986) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:904) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1289) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1198) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:93) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:362) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397) at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:189) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348) at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230) at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272) at org.apache.solr.handler.ReplicationHandler.inform(ReplicationHandler.java:1241) at org.apache.solr.cloud.ReplicateFromLeader.startReplication(ReplicateFromLeader.java:109) at org.apache.solr.cloud.ZkController.startReplicationFromLeader(ZkController.java:1323) at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:709) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:332) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:315) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1075) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:904) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1289) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1198) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:93) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:362) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397) at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:189) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348) at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230) at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420) at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:250) at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:678) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:332) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:315) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) expected null, but was:<ObjectTracker found 5 object(s) that were not released!!! [MMapDirectory, MMapDirectory, InternalHttpClient, SolrCore, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:519) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424) at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1210) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:800) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:986) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:904) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1289) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1198) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:93) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:362) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397) at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:189) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348) at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230) at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272) at org.apache.solr.handler.ReplicationHandler.inform(ReplicationHandler.java:1241) at org.apache.solr.cloud.ReplicateFromLeader.startReplication(ReplicateFromLeader.java:109) at org.apache.solr.cloud.ZkController.startReplicationFromLeader(ZkController.java:1323) at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:709) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:332) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:315) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1075) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:904) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1289) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1198) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:93) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:362) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397) at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:189) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348) at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230) at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420) at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:250) at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:678) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:332) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:315) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) >
Stack Trace:
java.lang.AssertionError: ObjectTracker found 5 object(s) that were not released!!! [MMapDirectory, MMapDirectory, InternalHttpClient, SolrCore, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:519)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424)
at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1210)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)
at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:800)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:986)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:904)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1289)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1198)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:93)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:362)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:189)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339)
at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348)
at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)
at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)
at org.apache.solr.handler.ReplicationHandler.inform(ReplicationHandler.java:1241)
at org.apache.solr.cloud.ReplicateFromLeader.startReplication(ReplicateFromLeader.java:109)
at org.apache.solr.cloud.ZkController.startReplicationFromLeader(ZkController.java:1323)
at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:709)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:332)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:315)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1075)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:904)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1289)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1198)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:93)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:362)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:189)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339)
at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348)
at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)
at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)
at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:250)
at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:678)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:332)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:315)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
expected null, but was:<ObjectTracker found 5 object(s) that were not released!!! [MMapDirectory, MMapDirectory, InternalHttpClient, SolrCore, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:519)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424)
at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1210)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)
at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:800)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:986)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:904)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1289)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1198)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:93)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:362)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:189)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339)
at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348)
at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)
at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)
at org.apache.solr.handler.ReplicationHandler.inform(ReplicationHandler.java:1241)
at org.apache.solr.cloud.ReplicateFromLeader.startReplication(ReplicateFromLeader.java:109)
at org.apache.solr.cloud.ZkController.startReplicationFromLeader(ZkController.java:1323)
at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:709)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:332)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:315)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1075)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:904)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1289)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1198)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:93)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:362)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:189)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339)
at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348)
at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)
at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)
at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:250)
at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:678)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:332)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:315)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
>
at __randomizedtesting.SeedInfo.seed([6FCF35F7EB24C8D4]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.failNotNull(Assert.java:755)
at org.junit.Assert.assertNull(Assert.java:737)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:333)
at sun.reflect.GeneratedMethodAccessor61.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:901)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 14045 lines...]
[junit4] Suite: org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore
[junit4] 2> 1470085 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to test-framework derived value of '/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/server/solr/configsets/_default/conf'
[junit4] 2> 1470085 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_6FCF35F7EB24C8D4-001/data-dir-110-001
[junit4] 2> 1470085 WARN (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=24 numCloses=24
[junit4] 2> 1470086 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 1470087 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 1470087 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 1470127 WARN (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1470129 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 1470130 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1470130 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1470130 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1470131 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7f59a8a2{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
[junit4] 2> 1470257 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@198a0f5b{hdfs,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/jetty-lucene2-us-west_apache_org-35466-hadoop-hdfs-3_2_0-tests_jar-_-any-1470310146895005246.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/hdfs}
[junit4] 2> 1470257 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5bc92cba{HTTP/1.1, (http/1.1)}{lucene2-us-west.apache.org:35466}
[junit4] 2> 1470257 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.Server Started @1470292ms
[junit4] 2> 1470303 WARN (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1470304 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 1470304 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1470304 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1470304 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1470305 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@110426e0{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
[junit4] 2> 1470427 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@2a74f59a{datanode,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/jetty-localhost-40957-hadoop-hdfs-3_2_0-tests_jar-_-any-2173619580192641496.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/datanode}
[junit4] 2> 1470428 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3c162d63{HTTP/1.1, (http/1.1)}{localhost:40957}
[junit4] 2> 1470428 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.Server Started @1470463ms
[junit4] 2> 1470910 WARN (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1470911 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 1470912 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1470912 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1470912 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1470912 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2ae0b30a{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
[junit4] 2> 1470975 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x6adf534d11e7c48c: Processing first storage report for DS-8b0454c0-3c13-450b-900c-6fb41872a143 from datanode c6aad607-ddd5-46d2-8e19-16ffbf00d440
[junit4] 2> 1470975 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x6adf534d11e7c48c: from storage DS-8b0454c0-3c13-450b-900c-6fb41872a143 node DatanodeRegistration(127.0.0.1:37889, datanodeUuid=c6aad607-ddd5-46d2-8e19-16ffbf00d440, infoPort=40016, infoSecurePort=0, ipcPort=44956, storageInfo=lv=-57;cid=testClusterID;nsid=688246582;c=1588448457228), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
[junit4] 2> 1470975 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x6adf534d11e7c48c: Processing first storage report for DS-f2a2fd2b-9bd9-4c57-91fd-2d90e703b296 from datanode c6aad607-ddd5-46d2-8e19-16ffbf00d440
[junit4] 2> 1470975 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x6adf534d11e7c48c: from storage DS-f2a2fd2b-9bd9-4c57-91fd-2d90e703b296 node DatanodeRegistration(127.0.0.1:37889, datanodeUuid=c6aad607-ddd5-46d2-8e19-16ffbf00d440, infoPort=40016, infoSecurePort=0, ipcPort=44956, storageInfo=lv=-57;cid=testClusterID;nsid=688246582;c=1588448457228), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
[junit4] 2> 1471046 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@5a42c2fe{datanode,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/jetty-localhost-38698-hadoop-hdfs-3_2_0-tests_jar-_-any-6626990524608344283.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/datanode}
[junit4] 2> 1471047 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.AbstractConnector Started ServerConnector@534cd616{HTTP/1.1, (http/1.1)}{localhost:38698}
[junit4] 2> 1471047 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.e.j.s.Server Started @1471082ms
[junit4] 2> 1471119 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xbf005f3584267ce3: Processing first storage report for DS-91764831-f28a-4b6d-8731-adfcda0f44da from datanode e94791e1-948e-4903-b9b8-0e0be280d328
[junit4] 2> 1471119 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xbf005f3584267ce3: from storage DS-91764831-f28a-4b6d-8731-adfcda0f44da node DatanodeRegistration(127.0.0.1:32783, datanodeUuid=e94791e1-948e-4903-b9b8-0e0be280d328, infoPort=40027, infoSecurePort=0, ipcPort=39661, storageInfo=lv=-57;cid=testClusterID;nsid=688246582;c=1588448457228), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
[junit4] 2> 1471120 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xbf005f3584267ce3: Processing first storage report for DS-fb5b3609-cf14-48b4-b228-485490474327 from datanode e94791e1-948e-4903-b9b8-0e0be280d328
[junit4] 2> 1471120 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xbf005f3584267ce3: from storage DS-fb5b3609-cf14-48b4-b228-485490474327 node DatanodeRegistration(127.0.0.1:32783, datanodeUuid=e94791e1-948e-4903-b9b8-0e0be280d328, infoPort=40027, infoSecurePort=0, ipcPort=39661, storageInfo=lv=-57;cid=testClusterID;nsid=688246582;c=1588448457228), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
[junit4] 2> 1471167 WARN (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.a.c.TestHdfsCloudBackupRestore The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 1476185 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_6FCF35F7EB24C8D4-001/tempDir-002
[junit4] 2> 1476186 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1476186 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port: 0.0.0.0/0.0.0.0:0
[junit4] 2> 1476186 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1476286 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port: 36481
[junit4] 2> 1476286 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:36481
[junit4] 2> 1476286 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:36481
[junit4] 2> 1476286 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 36481
[junit4] 2> 1476288 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1476291 INFO (zkConnectionManagerCallback-12732-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1476291 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1476295 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1476296 INFO (zkConnectionManagerCallback-12734-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1476296 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1476297 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1476298 INFO (zkConnectionManagerCallback-12736-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1476298 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1476401 WARN (jetty-launcher-12737-thread-1) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 1476401 INFO (jetty-launcher-12737-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 1476401 INFO (jetty-launcher-12737-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
[junit4] 2> 1476401 INFO (jetty-launcher-12737-thread-1) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 1476402 WARN (jetty-launcher-12737-thread-2) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 1476402 INFO (jetty-launcher-12737-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 1476402 INFO (jetty-launcher-12737-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
[junit4] 2> 1476402 INFO (jetty-launcher-12737-thread-2) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 1476404 INFO (jetty-launcher-12737-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1476404 INFO (jetty-launcher-12737-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1476404 INFO (jetty-launcher-12737-thread-2) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1476404 INFO (jetty-launcher-12737-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1476404 INFO (jetty-launcher-12737-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1476404 INFO (jetty-launcher-12737-thread-1) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1476404 INFO (jetty-launcher-12737-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@115bbc3a{/solr,null,AVAILABLE}
[junit4] 2> 1476404 INFO (jetty-launcher-12737-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@549a0e58{/solr,null,AVAILABLE}
[junit4] 2> 1476405 INFO (jetty-launcher-12737-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6dbb0f07{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:46637}
[junit4] 2> 1476405 INFO (jetty-launcher-12737-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@79876f1b{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:41850}
[junit4] 2> 1476405 INFO (jetty-launcher-12737-thread-2) [ ] o.e.j.s.Server Started @1476440ms
[junit4] 2> 1476405 INFO (jetty-launcher-12737-thread-1) [ ] o.e.j.s.Server Started @1476440ms
[junit4] 2> 1476405 INFO (jetty-launcher-12737-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46637}
[junit4] 2> 1476405 INFO (jetty-launcher-12737-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41850}
[junit4] 2> 1476405 ERROR (jetty-launcher-12737-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1476405 ERROR (jetty-launcher-12737-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1476405 INFO (jetty-launcher-12737-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1476405 INFO (jetty-launcher-12737-thread-2) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1476405 INFO (jetty-launcher-12737-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.6.0
[junit4] 2> 1476405 INFO (jetty-launcher-12737-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.6.0
[junit4] 2> 1476405 INFO (jetty-launcher-12737-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1476405 INFO (jetty-launcher-12737-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1476405 INFO (jetty-launcher-12737-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr
[junit4] 2> 1476405 INFO (jetty-launcher-12737-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr
[junit4] 2> 1476405 INFO (jetty-launcher-12737-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-05-02T19:41:03.531Z
[junit4] 2> 1476405 INFO (jetty-launcher-12737-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-05-02T19:41:03.531Z
[junit4] 2> 1476406 INFO (jetty-launcher-12737-thread-1) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1476406 INFO (jetty-launcher-12737-thread-2) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1476407 INFO (zkConnectionManagerCallback-12740-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1476407 INFO (jetty-launcher-12737-thread-1) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1476407 INFO (zkConnectionManagerCallback-12741-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1476407 INFO (jetty-launcher-12737-thread-2) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1476408 INFO (jetty-launcher-12737-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1476408 INFO (jetty-launcher-12737-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1476414 INFO (jetty-launcher-12737-thread-2) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@42582a60, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1476414 INFO (jetty-launcher-12737-thread-1) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@42582a60, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1477066 INFO (jetty-launcher-12737-thread-1) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 1477067 WARN (jetty-launcher-12737-thread-1) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@57e30390[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1477067 WARN (jetty-launcher-12737-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@57e30390[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1477070 WARN (jetty-launcher-12737-thread-1) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@1b5eda14[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1477070 WARN (jetty-launcher-12737-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@1b5eda14[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1477072 INFO (jetty-launcher-12737-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36481/solr
[junit4] 2> 1477073 INFO (jetty-launcher-12737-thread-1) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1477074 INFO (zkConnectionManagerCallback-12755-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1477074 INFO (jetty-launcher-12737-thread-1) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1477176 INFO (jetty-launcher-12737-thread-1) [n:127.0.0.1:41850_solr ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1477177 INFO (zkConnectionManagerCallback-12757-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1477177 INFO (jetty-launcher-12737-thread-1) [n:127.0.0.1:41850_solr ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1477250 INFO (jetty-launcher-12737-thread-1) [n:127.0.0.1:41850_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:41850_solr
[junit4] 2> 1477251 INFO (jetty-launcher-12737-thread-1) [n:127.0.0.1:41850_solr ] o.a.s.c.Overseer Overseer (id=72727634029707270-127.0.0.1:41850_solr-n_0000000000) starting
[junit4] 2> 1477257 INFO (OverseerStateUpdate-72727634029707270-127.0.0.1:41850_solr-n_0000000000) [n:127.0.0.1:41850_solr ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:41850_solr
[junit4] 2> 1477257 INFO (jetty-launcher-12737-thread-1) [n:127.0.0.1:41850_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41850_solr
[junit4] 2> 1477258 INFO (OverseerStateUpdate-72727634029707270-127.0.0.1:41850_solr-n_0000000000) [n:127.0.0.1:41850_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1477260 INFO (jetty-launcher-12737-thread-1) [n:127.0.0.1:41850_solr ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 1477261 WARN (jetty-launcher-12737-thread-1) [n:127.0.0.1:41850_solr ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 1477261 INFO (jetty-launcher-12737-thread-1) [n:127.0.0.1:41850_solr ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = hdfs,class = org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = {name=hdfs, class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = {location=/backup,solr.hdfs.home=hdfs://lucene2-us-west.apache.org:41309/solr,solr.hdfs.confdir=}}
[junit4] 2> 1477261 INFO (jetty-launcher-12737-thread-1) [n:127.0.0.1:41850_solr ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = poisioned,class = org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository,attributes = {default=true, name=poisioned, class=org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository},}
[junit4] 2> 1477261 INFO (jetty-launcher-12737-thread-1) [n:127.0.0.1:41850_solr ] o.a.s.c.b.r.BackupRepositoryFactory Default configuration for backup repository is with configuration params {type = repository,name = poisioned,class = org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository,attributes = {default=true, name=poisioned, class=org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository},}
[junit4] 2> 1477279 INFO (jetty-launcher-12737-thread-1) [n:127.0.0.1:41850_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1477308 INFO (jetty-launcher-12737-thread-1) [n:127.0.0.1:41850_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42582a60
[junit4] 2> 1477320 INFO (jetty-launcher-12737-thread-1) [n:127.0.0.1:41850_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42582a60
[junit4] 2> 1477320 INFO (jetty-launcher-12737-thread-1) [n:127.0.0.1:41850_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42582a60
[junit4] 2> 1477322 INFO (jetty-launcher-12737-thread-1) [n:127.0.0.1:41850_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_6FCF35F7EB24C8D4-001/tempDir-002/node1/.
[junit4] 2> 1477836 INFO (jetty-launcher-12737-thread-2) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 1477837 WARN (jetty-launcher-12737-thread-2) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@50311b08[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1477837 WARN (jetty-launcher-12737-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@50311b08[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1477850 WARN (jetty-launcher-12737-thread-2) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@590fe2a3[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1477850 WARN (jetty-launcher-12737-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@590fe2a3[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1477851 INFO (jetty-launcher-12737-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36481/solr
[junit4] 2> 1477852 INFO (jetty-launcher-12737-thread-2) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1477854 INFO (zkConnectionManagerCallback-12777-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1477854 INFO (jetty-launcher-12737-thread-2) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1477956 INFO (jetty-launcher-12737-thread-2) [n:127.0.0.1:46637_solr ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1477957 INFO (zkConnectionManagerCallback-12779-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1477957 INFO (jetty-launcher-12737-thread-2) [n:127.0.0.1:46637_solr ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1477963 INFO (jetty-launcher-12737-thread-2) [n:127.0.0.1:46637_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1477967 INFO (jetty-launcher-12737-thread-2) [n:127.0.0.1:46637_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:46637_solr as DOWN
[junit4] 2> 1477968 INFO (jetty-launcher-12737-thread-2) [n:127.0.0.1:46637_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1477968 INFO (jetty-launcher-12737-thread-2) [n:127.0.0.1:46637_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46637_solr
[junit4] 2> 1477969 INFO (zkCallback-12756-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1477969 INFO (zkCallback-12778-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1477970 INFO (jetty-launcher-12737-thread-2) [n:127.0.0.1:46637_solr ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 1477971 WARN (jetty-launcher-12737-thread-2) [n:127.0.0.1:46637_solr ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 1477971 INFO (jetty-launcher-12737-thread-2) [n:127.0.0.1:46637_solr ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = hdfs,class = org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = {name=hdfs, class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = {location=/backup,solr.hdfs.home=hdfs://lucene2-us-west.apache.org:41309/solr,solr.hdfs.confdir=}}
[junit4] 2> 1477971 INFO (jetty-launcher-12737-thread-2) [n:127.0.0.1:46637_solr ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = poisioned,class = org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository,attributes = {default=true, name=poisioned, class=org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository},}
[junit4] 2> 1477971 INFO (jetty-launcher-12737-thread-2) [n:127.0.0.1:46637_solr ] o.a.s.c.b.r.BackupRepositoryFactory Default configuration for backup repository is with configuration params {type = repository,name = poisioned,class = org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository,attributes = {default=true, name=poisioned, class=org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository},}
[junit4] 2> 1477998 INFO (jetty-launcher-12737-thread-2) [n:127.0.0.1:46637_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1478030 INFO (jetty-launcher-12737-thread-2) [n:127.0.0.1:46637_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42582a60
[junit4] 2> 1478048 INFO (jetty-launcher-12737-thread-2) [n:127.0.0.1:46637_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42582a60
[junit4] 2> 1478048 INFO (jetty-launcher-12737-thread-2) [n:127.0.0.1:46637_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42582a60
[junit4] 2> 1478050 INFO (jetty-launcher-12737-thread-2) [n:127.0.0.1:46637_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_6FCF35F7EB24C8D4-001/tempDir-002/node2/.
[junit4] 2> 1478069 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=2
[junit4] 2> 1478069 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1478071 INFO (zkConnectionManagerCallback-12792-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1478071 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1478072 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1478073 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[6FCF35F7EB24C8D4]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36481/solr ready
[junit4] 2> 1478147 INFO (TEST-TestHdfsCloudBackupRestore.testRestoreFailure-seed#[6FCF35F7EB24C8D4]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testRestoreFailure
[junit4] 2> 1478149 INFO (qtp1669131544-19556) [n:127.0.0.1:46637_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=1&collection.configName=conf1&maxShardsPerNode=3&name=hdfsbackuprestore_testfailure&nrtReplicas=1&action=CREATE&numShards=2&tlogReplicas=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1478152 INFO (OverseerThreadFactory-12764-thread-1-processing-n:127.0.0.1:41850_solr) [n:127.0.0.1:41850_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection hdfsbackuprestore_testfailure
[junit4] 2> 1478256 WARN (OverseerThreadFactory-12764-thread-1-processing-n:127.0.0.1:41850_solr) [n:127.0.0.1:41850_solr ] o.a.s.c.a.c.CreateCollectionCmd Specified number of replicas of 3 on collection hdfsbackuprestore_testfailure is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
[junit4] 2> 1478259 INFO (qtp1669131544-19554) [n:127.0.0.1:46637_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1478261 INFO (qtp2000723843-19555) [n:127.0.0.1:41850_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1478263 INFO (qtp1669131544-19557) [n:127.0.0.1:46637_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1478266 INFO (qtp2000723843-19558) [n:127.0.0.1:41850_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=2
[junit4] 2> 1478273 INFO (OverseerStateUpdate-72727634029707270-127.0.0.1:41850_solr-n_0000000000) [n:127.0.0.1:41850_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"hdfsbackuprestore_testfailure",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"hdfsbackuprestore_testfailure_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:41850/solr",
[junit4] 2> "node_name":"127.0.0.1:41850_solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 1478275 INFO (OverseerStateUpdate-72727634029707270-127.0.0.1:41850_solr-n_0000000000) [n:127.0.0.1:41850_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"hdfsbackuprestore_testfailure",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"hdfsbackuprestore_testfailure_shard1_replica_t2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:46637/solr",
[junit4] 2> "node_name":"127.0.0.1:46637_solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 1478277 INFO (OverseerStateUpdate-72727634029707270-127.0.0.1:41850_solr-n_0000000000) [n:127.0.0.1:41850_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"hdfsbackuprestore_testfailure",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"hdfsbackuprestore_testfailure_shard1_replica_p4",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:46637/solr",
[junit4] 2> "node_name":"127.0.0.1:46637_solr",
[junit4] 2> "type":"PULL",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 1478279 INFO (OverseerStateUpdate-72727634029707270-127.0.0.1:41850_solr-n_0000000000) [n:127.0.0.1:41850_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"hdfsbackuprestore_testfailure",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"hdfsbackuprestore_testfailure_shard2_replica_n6",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:41850/solr",
[junit4] 2> "node_name":"127.0.0.1:41850_solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 1478281 INFO (OverseerStateUpdate-72727634029707270-127.0.0.1:41850_solr-n_0000000000) [n:127.0.0.1:41850_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"hdfsbackuprestore_testfailure",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"hdfsbackuprestore_testfailure_shard2_replica_t8",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:46637/solr",
[junit4] 2> "node_name":"127.0.0.1:46637_solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 1478284 INFO (OverseerStateUpdate-72727634029707270-127.0.0.1:41850_solr-n_0000000000) [n:127.0.0.1:41850_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"hdfsbackuprestore_testfailure",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"hdfsbackuprestore_testfailure_shard2_replica_p10",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:41850/solr",
[junit4] 2> "node_name":"127.0.0.1:41850_solr",
[junit4] 2> "type":"PULL",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 1478486 INFO (qtp2000723843-19553) [n:127.0.0.1:41850_solr x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf1&newCollection=true&name=hdfsbackuprestore_testfailure_shard1_replica_n1&action=CREATE&numShards=2&collection=hdfsbackuprestore_testfailure&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1478486 INFO (qtp1669131544-19554) [n:127.0.0.1:46637_solr x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf1&newCollection=true&name=hdfsbackuprestore_testfailure_shard1_replica_t2&action=CREATE&numShards=2&collection=hdfsbackuprestore_testfailure&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 1478486 INFO (qtp2000723843-19553) [n:127.0.0.1:41850_solr x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1478487 INFO (qtp1669131544-19557) [n:127.0.0.1:46637_solr x:hdfsbackuprestore_testfailure_shard1_replica_p4 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf1&newCollection=true&name=hdfsbackuprestore_testfailure_shard1_replica_p4&action=CREATE&numShards=2&collection=hdfsbackuprestore_testfailure&shard=shard1&wt=javabin&version=2&replicaType=PULL
[junit4] 2> 1478487 INFO (qtp2000723843-19558) [n:127.0.0.1:41850_solr x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node9&collection.configName=conf1&newCollection=true&name=hdfsbackuprestore_testfailure_shard2_replica_n6&action=CREATE&numShards=2&collection=hdfsbackuprestore_testfailure&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1478488 INFO (qtp2000723843-19555) [n:127.0.0.1:41850_solr x:hdfsbackuprestore_testfailure_shard2_replica_p10 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node12&collection.configName=conf1&newCollection=true&name=hdfsbackuprestore_testfailure_shard2_replica_p10&action=CREATE&numShards=2&collection=hdfsbackuprestore_testfailure&shard=shard2&wt=javabin&version=2&replicaType=PULL
[junit4] 2> 1478488 INFO (qtp1669131544-19618) [n:127.0.0.1:46637_solr x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node11&collection.configName=conf1&newCollection=true&name=hdfsbackuprestore_testfailure_shard2_replica_t8&action=CREATE&numShards=2&collection=hdfsbackuprestore_testfailure&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 1478501 INFO (qtp2000723843-19555) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node12 x:hdfsbackuprestore_testfailure_shard2_replica_p10 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.6.0
[junit4] 2> 1478501 INFO (qtp1669131544-19618) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node11 x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.6.0
[junit4] 2> 1478501 INFO (qtp1669131544-19554) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node5 x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.6.0
[junit4] 2> 1478501 INFO (qtp2000723843-19553) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node3 x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.6.0
[junit4] 2> 1478501 INFO (qtp1669131544-19557) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node7 x:hdfsbackuprestore_testfailure_shard1_replica_p4 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.6.0
[junit4] 2> 1478501 INFO (qtp2000723843-19558) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.6.0
[junit4] 2> 1478520 INFO (qtp2000723843-19553) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node3 x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.s.IndexSchema Schema name=minimal
[junit4] 2> 1478520 INFO (qtp1669131544-19618) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node11 x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.s.IndexSchema Schema name=minimal
[junit4] 2> 1478520 INFO (qtp2000723843-19555) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node12 x:hdfsbackuprestore_testfailure_shard2_replica_p10 ] o.a.s.s.IndexSchema Schema name=minimal
[junit4] 2> 1478520 INFO (qtp1669131544-19557) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node7 x:hdfsbackuprestore_testfailure_shard1_replica_p4 ] o.a.s.s.IndexSchema Schema name=minimal
[junit4] 2> 1478520 INFO (qtp2000723843-19558) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.s.IndexSchema Schema name=minimal
[junit4] 2> 1478520 INFO (qtp1669131544-19554) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node5 x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.s.IndexSchema Schema name=minimal
[junit4] 2> 1478523 INFO (qtp2000723843-19558) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1478523 INFO (qtp2000723843-19553) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node3 x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1478523 INFO (qtp1669131544-19554) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node5 x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1478523 INFO (qtp2000723843-19555) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node12 x:hdfsbackuprestore_testfailure_shard2_replica_p10 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1478523 INFO (qtp2000723843-19558) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.c.CoreContainer Creating SolrCore 'hdfsbackuprestore_testfailure_shard2_replica_n6' using configuration from configset conf1, trusted=true
[junit4] 2> 1478523 INFO (qtp1669131544-19557) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node7 x:hdfsbackuprestore_testfailure_shard1_replica_p4 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1478523 INFO (qtp2000723843-19553) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node3 x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'hdfsbackuprestore_testfailure_shard1_replica_n1' using configuration from configset conf1, trusted=true
[junit4] 2> 1478523 INFO (qtp1669131544-19557) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node7 x:hdfsbackuprestore_testfailure_shard1_replica_p4 ] o.a.s.c.CoreContainer Creating SolrCore 'hdfsbackuprestore_testfailure_shard1_replica_p4' using configuration from configset conf1, trusted=true
[junit4] 2> 1478523 INFO (qtp1669131544-19618) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node11 x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1478523 INFO (qtp1669131544-19618) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node11 x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.c.CoreContainer Creating SolrCore 'hdfsbackuprestore_testfailure_shard2_replica_t8' using configuration from configset conf1, trusted=true
[junit4] 2> 1478524 INFO (qtp1669131544-19554) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node5 x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.c.CoreContainer Creating SolrCore 'hdfsbackuprestore_testfailure_shard1_replica_t2' using configuration from configset conf1, trusted=true
[junit4] 2> 1478524 INFO (qtp2000723843-19555) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node12 x:hdfsbackuprestore_testfailure_shard2_replica_p10 ] o.a.s.c.CoreContainer Creating SolrCore 'hdfsbackuprestore_testfailure_shard2_replica_p10' using configuration from configset conf1, trusted=true
[junit4] 2> 1478524 INFO (qtp1669131544-19557) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node7 x:hdfsbackuprestore_testfailure_shard1_replica_p4 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.hdfsbackuprestore_testfailure.shard1.replica_p4' (registry 'solr.core.hdfsbackuprestore_testfailure.shard1.replica_p4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42582a60
[junit4] 2> 1478524 INFO (qtp1669131544-19618) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node11 x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.hdfsbackuprestore_testfailure.shard2.replica_t8' (registry 'solr.core.hdfsbackuprestore_testfailure.shard2.replica_t8') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42582a60
[junit4] 2> 1478524 INFO (qtp2000723843-19553) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node3 x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.hdfsbackuprestore_testfailure.shard1.replica_n1' (registry 'solr.core.hdfsbackuprestore_testfailure.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42582a60
[junit4] 2> 1478524 INFO (qtp2000723843-19555) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node12 x:hdfsbackuprestore_testfailure_shard2_replica_p10 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.hdfsbackuprestore_testfailure.shard2.replica_p10' (registry 'solr.core.hdfsbackuprestore_testfailure.shard2.replica_p10') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42582a60
[junit4] 2> 1478524 INFO (qtp1669131544-19554) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node5 x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.hdfsbackuprestore_testfailure.shard1.replica_t2' (registry 'solr.core.hdfsbackuprestore_testfailure.shard1.replica_t2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42582a60
[junit4] 2> 1478524 INFO (qtp2000723843-19558) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.hdfsbackuprestore_testfailure.shard2.replica_n6' (registry 'solr.core.hdfsbackuprestore_testfailure.shard2.replica_n6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@42582a60
[junit4] 2> 1478524 INFO (qtp1669131544-19557) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node7 x:hdfsbackuprestore_testfailure_shard1_replica_p4 ] o.a.s.c.SolrCore [[hdfsbackuprestore_testfailure_shard1_replica_p4] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_6FCF35F7EB24C8D4-001/tempDir-002/node2/./hdfsbackuprestore_testfailure_shard1_replica_p4], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_6FCF35F7EB24C8D4-001/tempDir-002/node2/./hdfsbackuprestore_testfailure_shard1_replica_p4/data/]
[junit4] 2> 1478524 INFO (qtp2000723843-19553) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node3 x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.SolrCore [[hdfsbackuprestore_testfailure_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_6FCF35F7EB24C8D4-001/tempDir-002/node1/./hdfsbackuprestore_testfailure_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_6FCF35F7EB24C8D4-001/tempDir-002/node1/./hdfsbackuprestore_testfailure_shard1_replica_n1/data/]
[junit4] 2> 1478524 INFO (qtp1669131544-19618) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node11 x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.c.SolrCore [[hdfsbackuprestore_testfailure_shard2_replica_t8] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_6FCF35F7EB24C8D4-001/tempDir-002/node2/./hdfsbackuprestore_testfailure_shard2_replica_t8], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_6FCF35F7EB24C8D4-001/tempDir-002/node2/./hdfsbackuprestore_testfailure_shard2_replica_t8/data/]
[junit4] 2> 1478524 INFO (qtp2000723843-19555) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node12 x:hdfsbackuprestore_testfailure_shard2_replica_p10 ] o.a.s.c.SolrCore [[hdfsbackuprestore_testfailure_shard2_replica_p10] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_6FCF35F7EB24C8D4-001/tempDir-002/node1/./hdfsbackuprestore_testfailure_shard2_replica_p10], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_6FCF35F7EB24C8D4-001/tempDir-002/node1/./hdfsbackuprestore_testfailure_shard2_replica_p10/data/]
[junit4] 2> 1478524 INFO (qtp1669131544-19554) [n:127.0.0.1:46637_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node5 x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.c.SolrCore [[hdfsbackuprestore_testfailure_shard1_replica_t2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_6FCF35F7EB24C8D4-001/tempDir-002/node2/./hdfsbackuprestore_testfailure_shard1_replica_t2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_6FCF35F7EB24C8D4-001/tempDir-002/node2/./hdfsbackuprestore_testfailure_shard1_replica_t2/data/]
[junit4] 2> 1478524 INFO (qtp2000723843-19558) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.c.SolrCore [[hdfsbackuprestore_testfailure_shard2_replica_n6] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_6FCF35F7EB24C8D4-001/tempDir-002/node1/./hdfsbackuprestore_testfailure_shard2_replica_n6], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_6FCF35F7EB24C8D4-001/tempDir-002/node1/./hdfsbackuprestore_testfailure_shard2_replica_n6/data/]
[junit4] 2> 1478649 INFO (qtp2000723843-19553) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node3 x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1478649 INFO (qtp2000723843-19553) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node3 x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1478649 INFO (qtp2000723843-19558) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1478649 INFO (qtp2000723843-19558) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1478651 INFO (qtp2000723843-19553) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node3 x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1478651 INFO (qtp2000723843-19553) [n:127.0.0.1:41850_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node3 x:hdfsbackuprestore_testfailure_shard1_replica_
[...truncated too long message...]
ery(RecoveryStrategy.java:332)
[junit4] > at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:315)
[junit4] > at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
[junit4] > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > expected null, but was:<ObjectTracker found 5 object(s) that were not released!!! [MMapDirectory, MMapDirectory, InternalHttpClient, SolrCore, InternalHttpClient]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
[junit4] > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
[junit4] > at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:519)
[junit4] > at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
[junit4] > at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424)
[junit4] > at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1210)
[junit4] > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] > at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
[junit4] > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
[junit4] > at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)
[junit4] > at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:800)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:986)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:904)
[junit4] > at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1289)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1198)
[junit4] > at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:93)
[junit4] > at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:362)
[junit4] > at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
[junit4] > at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:189)
[junit4] > at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
[junit4] > at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339)
[junit4] > at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348)
[junit4] > at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)
[junit4] > at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)
[junit4] > at org.apache.solr.handler.ReplicationHandler.inform(ReplicationHandler.java:1241)
[junit4] > at org.apache.solr.cloud.ReplicateFromLeader.startReplication(ReplicateFromLeader.java:109)
[junit4] > at org.apache.solr.cloud.ZkController.startReplicationFromLeader(ZkController.java:1323)
[junit4] > at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:709)
[junit4] > at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:332)
[junit4] > at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:315)
[junit4] > at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
[junit4] > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1075)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:904)
[junit4] > at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1289)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1198)
[junit4] > at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:93)
[junit4] > at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:362)
[junit4] > at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
[junit4] > at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:189)
[junit4] > at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
[junit4] > at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339)
[junit4] > at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348)
[junit4] > at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)
[junit4] > at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)
[junit4] > at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)
[junit4] > at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:250)
[junit4] > at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:678)
[junit4] > at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:332)
[junit4] > at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:315)
[junit4] > at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
[junit4] > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > >
[junit4] > at __randomizedtesting.SeedInfo.seed([6FCF35F7EB24C8D4]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:333)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] Completed [299/907 (1!)] on J1 in 92.22s, 2 tests, 1 failure <<< FAILURES!
[...truncated 54711 lines...]