You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/12/15 05:30:18 UTC
[JENKINS] Lucene-Solr-BadApples-NightlyTests-master - Build # 41 -
Still Failing
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-master/41/
No tests ran.
Build Log:
[...truncated 6 lines...]
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
at hudson.scm.SCM.checkout(SCM.java:504)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1794)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout:
stderr: error: failed to write new configuration file .git/config.lock
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
at hudson.scm.SCM.checkout(SCM.java:504)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1794)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
> git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
> git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
at hudson.scm.SCM.checkout(SCM.java:504)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1794)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout:
stderr: error: failed to write new configuration file .git/config.lock
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
at hudson.scm.SCM.checkout(SCM.java:504)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1794)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
> git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
> git config remote.origin.url git://git.apache.org/lucene-solr.git # timeout=10
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from git://git.apache.org/lucene-solr.git
at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:888)
at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
at hudson.scm.SCM.checkout(SCM.java:504)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1794)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git config remote.origin.url git://git.apache.org/lucene-solr.git" returned status code 4:
stdout:
stderr: error: failed to write new configuration file .git/config.lock
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2002)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1970)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:1966)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1597)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommand(CliGitAPIImpl.java:1609)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.setRemoteUrl(CliGitAPIImpl.java:1243)
at hudson.plugins.git.GitAPI.setRemoteUrl(GitAPI.java:160)
at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at hudson.remoting.RemoteInvocationHandler$RPCRequest.perform(RemoteInvocationHandler.java:929)
at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:903)
at hudson.remoting.RemoteInvocationHandler$RPCRequest.call(RemoteInvocationHandler.java:855)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.remoting.RemoteInvocationHandler.invoke(RemoteInvocationHandler.java:283)
at com.sun.proxy.$Proxy117.setRemoteUrl(Unknown Source)
at org.jenkinsci.plugins.gitclient.RemoteGitImpl.setRemoteUrl(RemoteGitImpl.java:295)
at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:876)
at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1155)
at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1186)
at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
at hudson.scm.SCM.checkout(SCM.java:504)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1794)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-BadApples-NightlyTests-master #37 to compare, so performing full copy of artifacts
Recording test results
ERROR: Step ‘Publish JUnit test result report’ failed: Test reports were found but none of them are new. Did leafNodes run?
For example, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/build/core/test/TEST-org.apache.lucene.TestAssertions.xml is 6 days 15 hr old
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-BadApples-NightlyTests-master - Build # 56 -
Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-master/56/
4 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
ObjectTracker found 5 object(s) that were not released!!! [SolrCore, InternalHttpClient, MMapDirectory, MMapDirectory, MMapDirectory] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1063) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:883) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1193) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:699) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330) at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230) at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:418) at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237) at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422) at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1191) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:368) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:747) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:976) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:883) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1193) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:699) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422) at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237) at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) expected null, but was:<ObjectTracker found 5 object(s) that were not released!!! [SolrCore, InternalHttpClient, MMapDirectory, MMapDirectory, MMapDirectory] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1063) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:883) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1193) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:699) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330) at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230) at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:418) at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237) at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422) at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1191) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:368) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:747) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:976) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:883) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1193) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:699) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422) at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237) at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) >
Stack Trace:
java.lang.AssertionError: ObjectTracker found 5 object(s) that were not released!!! [SolrCore, InternalHttpClient, MMapDirectory, MMapDirectory, MMapDirectory]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1063)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:883)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1193)
at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:699)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321)
at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330)
at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)
at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:418)
at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)
at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1191)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:368)
at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:747)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:976)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:883)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1193)
at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:699)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)
at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
expected null, but was:<ObjectTracker found 5 object(s) that were not released!!! [SolrCore, InternalHttpClient, MMapDirectory, MMapDirectory, MMapDirectory]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1063)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:883)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1193)
at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:699)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321)
at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330)
at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)
at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:418)
at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)
at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
at org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1191)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:368)
at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:747)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:976)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:883)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1193)
at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:699)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:509)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:351)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:422)
at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237)
at org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
>
at __randomizedtesting.SeedInfo.seed([10CFE156AFBFCDCA]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.failNotNull(Assert.java:755)
at org.junit.Assert.assertNull(Assert.java:737)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:325)
at sun.reflect.GeneratedMethodAccessor74.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:901)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.basicTest
Error Message:
Timeout occurred while waiting response from server at: https://127.0.0.1:42640/solr
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: https://127.0.0.1:42640/solr
at __randomizedtesting.SeedInfo.seed([10CFE156AFBFCDCA:E23BF634EB1AC0F9]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:660)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1055)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:830)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:763)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:224)
at org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.basicTest(LeaderVoteWaitTimeoutTest.java:155)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
at sun.security.ssl.InputRecord.read(InputRecord.java:503)
at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:975)
at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:933)
at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:548)
... 49 more
FAILED: org.apache.solr.handler.component.TestDistributedStatsComponentCardinality.test
Error Message:
Captured an uncaught exception in thread: Thread[id=153660, name=Thread-98119, state=RUNNABLE, group=TGRP-TestDistributedStatsComponentCardinality]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=153660, name=Thread-98119, state=RUNNABLE, group=TGRP-TestDistributedStatsComponentCardinality]
at __randomizedtesting.SeedInfo.seed([10CFE156AFBFCDCA:989BDE8C0143A032]:0)
Caused by: junit.framework.AssertionFailedError: .responseHeader.status:200!=0
at __randomizedtesting.SeedInfo.seed([10CFE156AFBFCDCA]:0)
at junit.framework.Assert.fail(Assert.java:57)
at org.apache.solr.BaseDistributedSearchTestCase.compareSolrResponses(BaseDistributedSearchTestCase.java:999)
at org.apache.solr.BaseDistributedSearchTestCase.compareResponses(BaseDistributedSearchTestCase.java:1026)
at org.apache.solr.BaseDistributedSearchTestCase$5.run(BaseDistributedSearchTestCase.java:695)
FAILED: org.apache.solr.uninverting.TestDocTermOrdsUninvertLimit.testTriggerUnInvertLimit
Error Message:
refusing to delete any files: this IndexWriter hit an unrecoverable exception
Stack Trace:
org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
at org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:349)
at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:669)
at org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:589)
at org.apache.lucene.index.IndexFileDeleter.checkpoint(IndexFileDeleter.java:531)
at org.apache.lucene.index.IndexWriter.checkpoint(IndexWriter.java:2563)
at org.apache.lucene.index.IndexWriter.publishFlushedSegment(IndexWriter.java:2641)
at org.apache.lucene.index.IndexWriter.lambda$publishFlushedSegments$10(IndexWriter.java:5079)
at org.apache.lucene.index.DocumentsWriterFlushQueue.innerPurge(DocumentsWriterFlushQueue.java:115)
at org.apache.lucene.index.DocumentsWriterFlushQueue.tryPurge(DocumentsWriterFlushQueue.java:146)
at org.apache.lucene.index.DocumentsWriter.purgeFlushTickets(DocumentsWriter.java:201)
at org.apache.lucene.index.IndexWriter.publishFlushedSegments(IndexWriter.java:5059)
at org.apache.lucene.index.IndexWriter$1.afterSegmentsFlushed(IndexWriter.java:369)
at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:599)
at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:413)
at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:514)
at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1594)
at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:191)
at org.apache.solr.uninverting.TestDocTermOrdsUninvertLimit.testTriggerUnInvertLimit(TestDocTermOrdsUninvertLimit.java:65)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.OutOfMemoryError: Java heap space
at java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:57)
at java.nio.ByteBuffer.allocate(ByteBuffer.java:335)
at org.apache.lucene.store.ByteBuffersDataOutput$$Lambda$267/1503397182.apply(Unknown Source)
at org.apache.lucene.store.ByteBuffersDataOutput.appendBlock(ByteBuffersDataOutput.java:447)
at org.apache.lucene.store.ByteBuffersDataOutput.writeBytes(ByteBuffersDataOutput.java:164)
at org.apache.lucene.store.ByteBuffersIndexOutput.writeBytes(ByteBuffersIndexOutput.java:115)
at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:141)
at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
at org.apache.lucene.store.RateLimitedIndexOutput.writeByte(RateLimitedIndexOutput.java:66)
at org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:70)
at org.apache.lucene.codecs.CodecUtil.writeFooter(CodecUtil.java:391)
at org.apache.lucene.codecs.lucene50.Lucene50PostingsWriter.close(Lucene50PostingsWriter.java:504)
at org.apache.lucene.util.IOUtils.close(IOUtils.java:88)
at org.apache.lucene.util.IOUtils.close(IOUtils.java:76)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.close(BlockTreeTermsWriter.java:1026)
at org.apache.lucene.util.IOUtils.closeWhileHandlingException(IOUtils.java:122)
at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:175)
at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:244)
at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139)
at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4459)
at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4054)
at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
Build Log:
[...truncated 13209 lines...]
[junit4] Suite: org.apache.solr.cloud.LeaderVoteWaitTimeoutTest
[junit4] 2> 136488 INFO (SUITE-LeaderVoteWaitTimeoutTest-seed#[10CFE156AFBFCDCA]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderVoteWaitTimeoutTest_10CFE156AFBFCDCA-001/init-core-data-001
[junit4] 2> 136488 WARN (SUITE-LeaderVoteWaitTimeoutTest-seed#[10CFE156AFBFCDCA]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
[junit4] 2> 136488 INFO (SUITE-LeaderVoteWaitTimeoutTest-seed#[10CFE156AFBFCDCA]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 136492 INFO (SUITE-LeaderVoteWaitTimeoutTest-seed#[10CFE156AFBFCDCA]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 136496 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testMostInSyncReplicasCanWinElection
[junit4] 2> 136501 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderVoteWaitTimeoutTest_10CFE156AFBFCDCA-001/tempDir-001
[junit4] 2> 136501 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 136502 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 136502 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 136602 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.ZkTestServer start zk server on port:38663
[junit4] 2> 136602 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:38663
[junit4] 2> 136602 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 38663
[junit4] 2> 136611 INFO (zkConnectionManagerCallback-438-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 136618 INFO (zkConnectionManagerCallback-440-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 136621 INFO (zkConnectionManagerCallback-442-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 136629 WARN (jetty-launcher-443-thread-2) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 136630 WARN (jetty-launcher-443-thread-3) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 136630 INFO (jetty-launcher-443-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 136630 INFO (jetty-launcher-443-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 136630 WARN (jetty-launcher-443-thread-1) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 136630 INFO (jetty-launcher-443-thread-3) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 136630 INFO (jetty-launcher-443-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 136630 INFO (jetty-launcher-443-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 136631 INFO (jetty-launcher-443-thread-1) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 136631 INFO (jetty-launcher-443-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 136631 INFO (jetty-launcher-443-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 136631 INFO (jetty-launcher-443-thread-3) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 136631 INFO (jetty-launcher-443-thread-3) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 136631 INFO (jetty-launcher-443-thread-3) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 136632 INFO (jetty-launcher-443-thread-2) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 136632 INFO (jetty-launcher-443-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3978b02b{/solr,null,AVAILABLE}
[junit4] 2> 136635 WARN (jetty-launcher-443-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@62957234[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 136642 INFO (jetty-launcher-443-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@25a1f7c5{SSL,[ssl, http/1.1]}{127.0.0.1:39458}
[junit4] 2> 136642 INFO (jetty-launcher-443-thread-3) [ ] o.e.j.s.Server Started @136716ms
[junit4] 2> 136642 INFO (jetty-launcher-443-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39458}
[junit4] 2> 136643 ERROR (jetty-launcher-443-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 136643 INFO (jetty-launcher-443-thread-3) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 136643 INFO (jetty-launcher-443-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 136643 INFO (jetty-launcher-443-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 136644 INFO (jetty-launcher-443-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 136644 INFO (jetty-launcher-443-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-03-29T20:26:57.851Z
[junit4] 2> 136651 INFO (jetty-launcher-443-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 136651 INFO (jetty-launcher-443-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 136651 INFO (jetty-launcher-443-thread-1) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 136651 INFO (jetty-launcher-443-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@25c5dccc{/solr,null,AVAILABLE}
[junit4] 2> 136651 WARN (jetty-launcher-443-thread-4) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 136652 WARN (jetty-launcher-443-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@46b9512b[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 136652 INFO (jetty-launcher-443-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 136652 INFO (jetty-launcher-443-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 136652 INFO (jetty-launcher-443-thread-4) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 136662 INFO (jetty-launcher-443-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 136662 INFO (jetty-launcher-443-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 136662 INFO (jetty-launcher-443-thread-2) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 136662 INFO (jetty-launcher-443-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4758a83a{/solr,null,AVAILABLE}
[junit4] 2> 136663 WARN (jetty-launcher-443-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@57c4bdb6[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 136666 INFO (jetty-launcher-443-thread-4) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 136666 INFO (jetty-launcher-443-thread-4) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 136666 INFO (jetty-launcher-443-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@69faec5a{SSL,[ssl, http/1.1]}{127.0.0.1:33713}
[junit4] 2> 136666 INFO (jetty-launcher-443-thread-1) [ ] o.e.j.s.Server Started @136740ms
[junit4] 2> 136666 INFO (jetty-launcher-443-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33713}
[junit4] 2> 136667 ERROR (jetty-launcher-443-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 136667 INFO (jetty-launcher-443-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 136667 INFO (jetty-launcher-443-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 136667 INFO (jetty-launcher-443-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 136667 INFO (jetty-launcher-443-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 136667 INFO (jetty-launcher-443-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-03-29T20:26:57.874Z
[junit4] 2> 136668 INFO (jetty-launcher-443-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4256f5c5{SSL,[ssl, http/1.1]}{127.0.0.1:43777}
[junit4] 2> 136668 INFO (jetty-launcher-443-thread-2) [ ] o.e.j.s.Server Started @136742ms
[junit4] 2> 136668 INFO (jetty-launcher-443-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43777}
[junit4] 2> 136668 ERROR (jetty-launcher-443-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 136668 INFO (jetty-launcher-443-thread-2) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 136669 INFO (jetty-launcher-443-thread-4) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 136669 INFO (jetty-launcher-443-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 136669 INFO (jetty-launcher-443-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 136669 INFO (jetty-launcher-443-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 136669 INFO (jetty-launcher-443-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-03-29T20:26:57.876Z
[junit4] 2> 136673 INFO (zkConnectionManagerCallback-445-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 136676 INFO (jetty-launcher-443-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6b58c4fb{/solr,null,AVAILABLE}
[junit4] 2> 136680 INFO (jetty-launcher-443-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 136688 WARN (jetty-launcher-443-thread-4) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@2d3c85df[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 136691 INFO (jetty-launcher-443-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@34e55d9b{SSL,[ssl, http/1.1]}{127.0.0.1:40120}
[junit4] 2> 136691 INFO (jetty-launcher-443-thread-4) [ ] o.e.j.s.Server Started @136765ms
[junit4] 2> 136691 INFO (jetty-launcher-443-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40120}
[junit4] 2> 136692 ERROR (jetty-launcher-443-thread-4) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 136692 INFO (jetty-launcher-443-thread-4) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 136692 INFO (jetty-launcher-443-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 136692 INFO (jetty-launcher-443-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 136692 INFO (jetty-launcher-443-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 136692 INFO (jetty-launcher-443-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-03-29T20:26:57.899Z
[junit4] 2> 136696 INFO (zkConnectionManagerCallback-447-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 136705 INFO (jetty-launcher-443-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 136714 INFO (zkConnectionManagerCallback-449-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 136721 INFO (jetty-launcher-443-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 136723 INFO (zkConnectionManagerCallback-451-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 136735 INFO (jetty-launcher-443-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 136876 INFO (jetty-launcher-443-thread-4) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 136877 WARN (jetty-launcher-443-thread-4) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 136947 INFO (jetty-launcher-443-thread-3) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 136948 WARN (jetty-launcher-443-thread-3) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 136954 WARN (jetty-launcher-443-thread-4) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@793c48d1[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 136969 INFO (jetty-launcher-443-thread-1) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 136977 WARN (jetty-launcher-443-thread-1) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 136983 WARN (jetty-launcher-443-thread-4) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 136989 WARN (jetty-launcher-443-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@364d0c88[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 137015 WARN (jetty-launcher-443-thread-3) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 137028 WARN (jetty-launcher-443-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@64f299a6[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 137041 WARN (jetty-launcher-443-thread-4) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@1a98c11b[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 137042 INFO (jetty-launcher-443-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38663/solr
[junit4] 2> 137044 WARN (jetty-launcher-443-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@7e8980a[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 137045 INFO (jetty-launcher-443-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38663/solr
[junit4] 2> 137053 WARN (jetty-launcher-443-thread-1) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 137099 WARN (jetty-launcher-443-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@474030c1[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 137100 INFO (jetty-launcher-443-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38663/solr
[junit4] 2> 137112 INFO (zkConnectionManagerCallback-466-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 137114 INFO (zkConnectionManagerCallback-469-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 137116 INFO (zkConnectionManagerCallback-473-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 137119 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x1037f9e26a50009, likely client has closed socket
[junit4] 2> 137121 INFO (zkConnectionManagerCallback-477-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 137125 INFO (zkConnectionManagerCallback-475-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 137137 INFO (zkConnectionManagerCallback-479-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 137162 INFO (jetty-launcher-443-thread-1) [n:127.0.0.1:33713_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33713_solr
[junit4] 2> 137162 INFO (jetty-launcher-443-thread-1) [n:127.0.0.1:33713_solr ] o.a.s.c.Overseer Overseer (id=73042336197967884-127.0.0.1:33713_solr-n_0000000000) starting
[junit4] 2> 137182 INFO (zkConnectionManagerCallback-490-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 137199 INFO (jetty-launcher-443-thread-1) [n:127.0.0.1:33713_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38663/solr ready
[junit4] 2> 137218 INFO (OverseerStateUpdate-73042336197967884-127.0.0.1:33713_solr-n_0000000000) [n:127.0.0.1:33713_solr ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:33713_solr
[junit4] 2> 137229 INFO (jetty-launcher-443-thread-1) [n:127.0.0.1:33713_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33713_solr
[junit4] 2> 137240 INFO (zkCallback-478-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 137242 INFO (zkCallback-489-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 137317 INFO (jetty-launcher-443-thread-3) [n:127.0.0.1:39458_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 137322 INFO (jetty-launcher-443-thread-3) [n:127.0.0.1:39458_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:39458_solr as DOWN
[junit4] 2> 137324 INFO (jetty-launcher-443-thread-3) [n:127.0.0.1:39458_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 137324 INFO (jetty-launcher-443-thread-3) [n:127.0.0.1:39458_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39458_solr
[junit4] 2> 137325 INFO (zkCallback-478-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 137325 INFO (zkCallback-489-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 137329 INFO (zkCallback-476-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 137363 INFO (jetty-launcher-443-thread-2) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 137364 WARN (jetty-launcher-443-thread-2) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 137365 WARN (jetty-launcher-443-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@7a4047a2[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 137371 WARN (jetty-launcher-443-thread-2) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 137375 WARN (jetty-launcher-443-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@6a7e09c8[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 137376 INFO (jetty-launcher-443-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38663/solr
[junit4] 2> 137376 INFO (jetty-launcher-443-thread-1) [n:127.0.0.1:33713_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 137398 INFO (jetty-launcher-443-thread-4) [n:127.0.0.1:40120_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 137398 INFO (zkConnectionManagerCallback-496-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 137403 INFO (jetty-launcher-443-thread-4) [n:127.0.0.1:40120_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:40120_solr as DOWN
[junit4] 2> 137404 INFO (jetty-launcher-443-thread-4) [n:127.0.0.1:40120_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 137404 INFO (jetty-launcher-443-thread-4) [n:127.0.0.1:40120_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40120_solr
[junit4] 2> 137405 INFO (zkCallback-478-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 137405 INFO (zkCallback-489-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 137429 INFO (zkCallback-476-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 137429 INFO (jetty-launcher-443-thread-3) [n:127.0.0.1:39458_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 137430 INFO (zkCallback-474-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 137430 INFO (jetty-launcher-443-thread-3) [n:127.0.0.1:39458_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38663/solr ready
[junit4] 2> 137436 INFO (zkConnectionManagerCallback-502-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 137449 INFO (jetty-launcher-443-thread-3) [n:127.0.0.1:39458_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 137451 INFO (jetty-launcher-443-thread-1) [n:127.0.0.1:33713_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33713.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38518cd9
[junit4] 2> 137467 INFO (jetty-launcher-443-thread-1) [n:127.0.0.1:33713_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33713.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38518cd9
[junit4] 2> 137467 INFO (jetty-launcher-443-thread-1) [n:127.0.0.1:33713_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33713.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38518cd9
[junit4] 2> 137469 INFO (jetty-launcher-443-thread-1) [n:127.0.0.1:33713_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderVoteWaitTimeoutTest_10CFE156AFBFCDCA-001/tempDir-001/node1/.
[junit4] 2> 137511 INFO (jetty-launcher-443-thread-3) [n:127.0.0.1:39458_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39458.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38518cd9
[junit4] 2> 137520 INFO (zkConnectionManagerCallback-508-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 137534 INFO (jetty-launcher-443-thread-4) [n:127.0.0.1:40120_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 137534 INFO (jetty-launcher-443-thread-3) [n:127.0.0.1:39458_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39458.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38518cd9
[junit4] 2> 137535 INFO (jetty-launcher-443-thread-3) [n:127.0.0.1:39458_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39458.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38518cd9
[junit4] 2> 137536 INFO (jetty-launcher-443-thread-4) [n:127.0.0.1:40120_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38663/solr ready
[junit4] 2> 137536 INFO (jetty-launcher-443-thread-3) [n:127.0.0.1:39458_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderVoteWaitTimeoutTest_10CFE156AFBFCDCA-001/tempDir-001/node3/.
[junit4] 2> 137540 INFO (jetty-launcher-443-thread-4) [n:127.0.0.1:40120_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 137542 INFO (zkConnectionManagerCallback-510-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 137548 INFO (jetty-launcher-443-thread-2) [n:127.0.0.1:43777_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 137552 INFO (jetty-launcher-443-thread-2) [n:127.0.0.1:43777_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:43777_solr as DOWN
[junit4] 2> 137553 INFO (jetty-launcher-443-thread-2) [n:127.0.0.1:43777_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 137553 INFO (jetty-launcher-443-thread-2) [n:127.0.0.1:43777_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43777_solr
[junit4] 2> 137556 INFO (zkCallback-478-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 137556 INFO (zkCallback-476-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 137556 INFO (zkCallback-474-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 137556 INFO (zkCallback-489-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 137570 INFO (jetty-launcher-443-thread-4) [n:127.0.0.1:40120_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40120.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38518cd9
[junit4] 2> 137574 INFO (zkCallback-495-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 137577 INFO (zkCallback-509-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 137582 INFO (zkCallback-507-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 137596 INFO (jetty-launcher-443-thread-4) [n:127.0.0.1:40120_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40120.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38518cd9
[junit4] 2> 137596 INFO (jetty-launcher-443-thread-4) [n:127.0.0.1:40120_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40120.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38518cd9
[junit4] 2> 137597 INFO (jetty-launcher-443-thread-4) [n:127.0.0.1:40120_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderVoteWaitTimeoutTest_10CFE156AFBFCDCA-001/tempDir-001/node4/.
[junit4] 2> 137602 INFO (zkConnectionManagerCallback-518-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 137604 INFO (jetty-launcher-443-thread-2) [n:127.0.0.1:43777_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 137605 INFO (jetty-launcher-443-thread-2) [n:127.0.0.1:43777_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38663/solr ready
[junit4] 2> 137609 INFO (jetty-launcher-443-thread-2) [n:127.0.0.1:43777_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 137633 INFO (jetty-launcher-443-thread-2) [n:127.0.0.1:43777_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43777.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38518cd9
[junit4] 2> 137661 INFO (jetty-launcher-443-thread-2) [n:127.0.0.1:43777_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43777.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38518cd9
[junit4] 2> 137661 INFO (jetty-launcher-443-thread-2) [n:127.0.0.1:43777_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43777.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38518cd9
[junit4] 2> 137663 INFO (jetty-launcher-443-thread-2) [n:127.0.0.1:43777_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderVoteWaitTimeoutTest_10CFE156AFBFCDCA-001/tempDir-001/node2/.
[junit4] 2> 137747 INFO (zkConnectionManagerCallback-524-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 137749 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 137751 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38663/solr ready
[junit4] 2> 137762 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=2015165875
[junit4] 2> 137762 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:33713_solr
[junit4] 2> 137763 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.CoreContainer Publish this node as DOWN...
[junit4] 2> 137763 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.ZkController Publish node=127.0.0.1:33713_solr as DOWN
[junit4] 2> 137765 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 137765 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 137766 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@46771726: rootName = solr_33713, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@5e8afa0a
[junit4] 2> 137773 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 137773 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6c06fab2: rootName = solr_33713, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@5aa4ef9d
[junit4] 2> 137777 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 137777 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7a1642f6: rootName = solr_33713, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@31f5a06
[junit4] 2> 137778 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 137778 INFO (closeThreadPool-527-thread-2) [ ] o.a.s.c.Overseer Overseer (id=73042336197967884-127.0.0.1:33713_solr-n_0000000000) closing
[junit4] 2> 137785 INFO (OverseerStateUpdate-73042336197967884-127.0.0.1:33713_solr-n_0000000000) [n:127.0.0.1:33713_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:33713_solr
[junit4] 2> 137786 INFO (OverseerAutoScalingTriggerThread-73042336197967884-127.0.0.1:33713_solr-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 137791 INFO (closeThreadPool-527-thread-1) [ ] o.a.s.c.Overseer Overseer (id=73042336197967884-127.0.0.1:33713_solr-n_0000000000) closing
[junit4] 2> 137791 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.Overseer Overseer (id=73042336197967884-127.0.0.1:33713_solr-n_0000000000) closing
[junit4] 2> 137792 INFO (zkCallback-474-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 137792 INFO (zkCallback-507-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 137794 INFO (zkCallback-523-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 137794 INFO (zkCallback-476-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 137794 INFO (zkCallback-476-thread-1) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:39458_solr
[junit4] 2> 137794 INFO (zkCallback-495-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 137795 INFO (zkCallback-509-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 137795 INFO (zkCallback-517-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 137796 INFO (zkCallback-476-thread-1) [n:127.0.0.1:39458_solr ] o.a.s.c.Overseer Overseer (id=73042336197967883-127.0.0.1:39458_solr-n_0000000001) starting
[junit4] 2> 137800 INFO (OverseerStateUpdate-73042336197967883-127.0.0.1:39458_solr-n_0000000001) [n:127.0.0.1:39458_solr ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:39458_solr
[junit4] 2> 137806 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@69faec5a{SSL,[ssl, http/1.1]}{127.0.0.1:0}
[junit4] 2> 137807 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@25c5dccc{/solr,null,UNAVAILABLE}
[junit4] 2> 137807 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.e.j.s.session node0 Stopped scavenging
[junit4] 2> 137808 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 137810 WARN (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 137810 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 137811 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 137813 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 137813 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 137813 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 137814 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5cb2b4fe{/solr,null,AVAILABLE}
[junit4] 2> 137815 WARN (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@201b0e6d[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 137815 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5dada609{SSL,[ssl, http/1.1]}{127.0.0.1:41388}
[junit4] 2> 137815 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.e.j.s.Server Started @137889ms
[junit4] 2> 137815 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=44399}
[junit4] 2> 137816 ERROR (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 137816 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 137816 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 137816 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 137816 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 137816 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-03-29T20:26:59.023Z
[junit4] 2> 137821 INFO (zkConnectionManagerCallback-529-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 137830 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 137987 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 137987 WARN (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 137990 WARN (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@3e070906[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 137993 WARN (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 137994 WARN (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@1232989d[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 137995 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38663/solr
[junit4] 2> 137999 INFO (zkConnectionManagerCallback-536-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 138004 INFO (zkConnectionManagerCallback-538-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 138010 INFO (TEST-LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection-seed#[10CFE156AFBFCDCA]) [n:127.0.0.1:44399_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 138014 INFO (TEST-LeaderVoteWaitTimeoutTest.test
[...truncated too long message...]
.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
[junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene80): {field=Lucene50(blocksize=128)}, docValues:{}, maxPointsInLeafNode=161, maxMBSortInHeap=6.99349066987206, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@1e232180), locale=de-GR, timezone=Australia/Tasmania
[junit4] 2> NOTE: Linux 4.4.0-137-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=59718496,total=477626368
[junit4] 2> NOTE: All tests run in this JVM: [SolrPluginUtilsTest, TestSolrCoreParser, IndexSchemaTest, MultiThreadedOCPTest, SolrCloudExampleTest, TriggerEventQueueTest, FullHLLTest, DistributedFacetPivotLongTailTest, TestConfigSets, TestReplicationHandlerBackup, ReturnFieldsTest, LeaderVoteWaitTimeoutTest, TestObjectReleaseTracker, ReplicaListTransformerTest, MinimalSchemaTest, Tagger2Test, TestFieldCacheWithThreads, TestRandomFaceting, TestMultiValuedNumericRangeQuery, TestJsonFacetRefinement, HdfsDirectoryTest, DistributedExpandComponentTest, TolerantUpdateProcessorTest, TestTrieFacet, HdfsWriteToMultipleCollectionsTest, DistributedUpdateProcessorTest, StatsComponentTest, TestBlendedInfixSuggestions, TestTolerantSearch, TestSkipOverseerOperations, DistanceFunctionTest, DisMaxRequestHandlerTest, SolrMetricReporterTest, ResponseHeaderTest, HdfsBasicDistributedZk2Test, TestRTGBase, NodeLostTriggerIntegrationTest, TestFieldCacheSort, TestUnInvertedFieldException, AliasIntegrationTest, BooleanFieldTest, TestCloudSchemaless, TestSubQueryTransformerCrossCore, SolrGraphiteReporterTest, TestManagedResourceStorage, TestExclusionRuleCollectionAccess, TestScoreJoinQPScore, TestSimGenericDistributedQueue, BlockJoinFacetRandomTest, TestComplexPhraseLeadingWildcard, TestMinMaxOnMultiValuedField, QueryEqualityTest, NumericFieldsTest, TestNumericTerms64, TestClusterProperties, TestInfoStreamLogging, TestInPlaceUpdatesRequiredField, TestConfigOverlay, PreAnalyzedFieldManagedSchemaCloudTest, SpatialHeatmapFacetsTest, TestEmbeddedSolrServerConstructors, WrapperMergePolicyFactoryTest, CoreAdminRequestStatusTest, TestBinaryResponseWriter, TestShardHandlerFactory, BasicDistributedZk2Test, TestNamedUpdateProcessors, HdfsRecoveryZkTest, TestNonDefinedSimilarityFactory, TestSolrCLIRunExample, UnloadDistributedZkTest, TestBlobHandler, TestJsonFacetsWithNestedObjects, BlobRepositoryCloudTest, DirectUpdateHandlerTest, LeaderElectionContextKeyTest, TestFieldCollectionResource, BigEndianAscendingWordSerializerTest, RequiredFieldsTest, RequestHandlersTest, DistributedFacetPivotWhiteBoxTest, TestStressLucene, TestSolrCloudSnapshots, TestPHPSerializedResponseWriter, TestDistribIDF, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestUniqueKeyFieldResource, TestLazyCores, TestSolrDeletionPolicy1, DistributedSuggestComponentTest, TestPivotHelperCode, CircularListTest, LeaderFailoverAfterPartitionTest, TestXmlQParserPlugin, TestSolrCloudWithHadoopAuthPlugin, TestExceedMaxTermLength, TestReRankQParserPlugin, TestCustomSort, TestSchemaSimilarityResource, TestOverriddenPrefixQueryForCustomFieldType, TestInitQParser, CdcrVersionReplicationTest, RestartWhileUpdatingTest, DeleteLastCustomShardedReplicaTest, EnumFieldTest, ProtectedTermFilterFactoryTest, TestFieldTypeResource, CreateCollectionCleanupTest, OrderedExecutorTest, SolrCoreCheckLockOnStartupTest, SpellingQueryConverterTest, UniqFieldsUpdateProcessorFactoryTest, TestDistributedSearch, TestLegacyFieldReuse, BasicFunctionalityTest, TestExtendedDismaxParser, StatelessScriptUpdateProcessorFactoryTest, MigrateRouteKeyTest, BlockCacheTest, TestLRUStatsCache, TestIntervalFaceting, PeerSyncTest, HttpPartitionOnCommitTest, DirectSolrConnectionTest, TestHalfAndHalfDocValues, TestLocalFSCloudBackupRestore, HdfsCollectionsAPIDistributedZkTest, SecurityConfHandlerTest, CollectionsAPIDistributedZkTest, MetricsConfigTest, TestLMDirichletSimilarityFactory, CurrencyFieldTypeTest, SoftAutoCommitTest, CachingDirectoryFactoryTest, TestOnReconnectListenerSupport, ReplaceNodeNoTargetTest, RecoveryZkTest, TestUseDocValuesAsStored, SampleTest, SolrInfoBeanTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestDistributedGrouping, TestGroupingSearch, TestHighlightDedupGrouping, TestJoin, TestDeprecatedFilters, TestLuceneMatchVersion, TestReversedWildcardFilterFactory, TestWordDelimiterFilterFactory, TokenizerChainTest, TestLuceneIndexBackCompat, TestJettySolrRunner, ConnectionReuseTest, ActionThrottleTest, BasicZkTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, ChaosMonkeySafeLeaderWithPullReplicasTest, ConcurrentCreateRoutedAliasTest, ConfigSetsAPITest, DeleteReplicaTest, DeleteStatusTest, DistribDocExpirationUpdateProcessorTest, DistribJoinFromCollectionTest, DistributedQueueTest, DistributedVersionInfoTest, ForceLeaderWithTlogReplicasTest, MoveReplicaHDFSFailoverTest, MoveReplicaTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, OverseerRolesTest, OverseerStatusTest, OverseerTaskQueueTest, OverseerTest, SSLMigrationTest, SaslZkACLProviderTest, ShardRoutingCustomTest, ShardRoutingTest, SharedFSAutoReplicaFailoverTest, TestCloudDeleteByQuery, TestCloudInspectUtil, TestCloudPhrasesIdentificationComponent, TestCloudPivotFacet, TestCloudPseudoReturnFields, TestCloudRecovery, TestCloudRecovery2, TestDeleteCollectionOnDownNodes, TestDistribDocBasedVersion, TestLeaderElectionZkExpiry, TestLockTree, TestPullReplica, TestPullReplicaErrorHandling, TestRandomRequestDistribution, TestRebalanceLeaders, TestRequestForwarding, TestSSLRandomization, TestSegmentSorting, TestShortCircuitedRequests, TestSolrCloudWithKerberosAlt, TestSolrCloudWithSecureImpersonation, TestStressCloudBlindAtomicUpdates, TestTlogReplica, TestTolerantUpdateProcessorCloud, TestTolerantUpdateProcessorRandomCloud, TestUtilizeNode, TestWithCollection, ZkCLITest, ZkControllerTest, ZkFailoverTest, ZkSolrClientTest, SimpleCollectionCreateDeleteTest, TestCollectionAPI, TestRequestStatusCollectionAPI, AutoAddReplicasPlanActionTest, AutoScalingHandlerTest, ComputePlanActionTest, HttpTriggerListenerTest, IndexSizeTriggerTest, MetricTriggerTest, NodeAddedTriggerIntegrationTest, RestoreTriggerStateTest, SearchRateTriggerIntegrationTest, TriggerSetPropertiesIntegrationTest, TestSimClusterStateProvider, TestSimExecutePlanAction, TestSimNodeAddedTrigger, TestSimNodeLostTrigger, TestSimPolicyCloud, TestSimTriggerIntegration, CdcrOpsAndBoundariesTest, HdfsChaosMonkeySafeLeaderTest, HdfsThreadLeakTest, HdfsTlogReplayBufferedWhileIndexingTest, TestSolrConfigHandlerConcurrent, XsltUpdateRequestHandlerTest, AutoscalingHistoryHandlerTest, MetricsHandlerTest, MetricsHistoryHandlerTest, ShowFileRequestHandlerTest, SystemInfoHandlerTest, TestApiFramework, TestCollectionAPIs, TestConfigsApi, TestCoreAdminApis, ZookeeperStatusHandlerTest, CustomTermsComponentTest, DistributedQueryComponentOptimizationTest, FacetPivotSmallTest, PhrasesIdentificationComponentTest, SearchHandlerTest, TermVectorComponentDistributedTest, TestDistributedStatsComponentCardinality, SolrRrdBackendFactoryTest, SubstringBytesRefFilterTest, TestFacetMethods, TestMacroExpander, TestMacros, JSONWriterTest, TestCustomDocTransformer, TestGraphMLResponseWriter, TestRawTransformer, TestSolrQueryResponse, TestChildDocTransformer, TestBulkSchemaAPI, DocValuesMissingTest, TestBulkSchemaConcurrent, TestSchemaManager, TestSchemalessBufferedUpdates, UUIDFieldTest, AnalyticsMergeStrategyTest, AnalyticsQueryTest, CurrencyRangeFacetCloudTest, CursorMarkTest, LargeFieldTest, SortSpecParsingTest, SpatialFilterTest, TestAddFieldRealTimeGet, TestFieldSortValues, TestFilteredDocIdSet, TestFoldingMultitermQuery, TestGraphTermsQParserPlugin, TestHashQParserPlugin, TestLRUCache, TestLegacyNumericRangeQueryBuilder, TestMaxScoreQueryParser, TestMinHashQParser, TestMissingGroups, TestRecoveryHdfs, TestSearchPerf, TestSearcherReuse, TestSimpleQParserPlugin, TestSmileRequest, TestSolr4Spatial, TestSolrJ, TestSolrQueryParser, TestSort, TestStressReorder, GraphQueryTest, TestCloudNestedDocsSort, TestBM25SimilarityFactory, TestDFISimilarityFactory, TestPerFieldSimilarityWithDefaultOverride, BasicAuthStandaloneTest, JWTAuthPluginTest, PKIAuthenticationIntegrationTest, TestAuthorizationFramework, SpellCheckCollatorTest, SuggesterFSTTest, SuggesterTSTTest, SuggesterWFSTTest, TestFreeTextSuggestions, TestHighFrequencyDictionaryFactory, TestDocTermOrdsUninvertLimit]
[junit4] Completed [827/846 (4!)] on J1 in 42.50s, 1 test, 1 error <<< FAILURES!
[...truncated 54 lines...]
[junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20190329_202441_0308179689234255985263.sysout
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/heapdumps/java_pid30929.hprof ...
[junit4] Heap dump file created [474635848 bytes in 1.168 secs]
[junit4] <<< JVM J1: EOF ----
[...truncated 8960 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/build.xml:661: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid30929.hprof
Total time: 295 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-BadApples-NightlyTests-master - Build # 55 -
Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-master/55/
4 tests failed.
FAILED: org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey
Error Message:
Address already in use
Stack Trace:
java.net.BindException: Address already in use
at __randomizedtesting.SeedInfo.seed([422049B263044CDA:C9079A632202E75E]:0)
at sun.nio.ch.Net.bind0(Native Method)
at sun.nio.ch.Net.bind(Net.java:433)
at sun.nio.ch.Net.bind(Net.java:425)
at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
at org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:342)
at org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:308)
at org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
at org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:236)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.eclipse.jetty.server.Server.doStart(Server.java:394)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:558)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:497)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:465)
at org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:499)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest
Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest: 1) Thread[id=57339, name=qtp846514698-57339, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=57336, name=qtp846514698-57336, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$27/644451172.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=57343, name=Scheduler-654441508, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=57341, name=qtp846514698-57341, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=57338, name=qtp846514698-57338, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=57340, name=qtp846514698-57340, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=57337, name=qtp846514698-57337-acceptor-0@3d5d18ec-ServerConnector@60323d6f{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:38927}, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=57342, name=qtp846514698-57342, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=57335, name=qtp846514698-57335, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$27/644451172.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=57458, name=Scheduler-910903452, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest:
1) Thread[id=57339, name=qtp846514698-57339, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=57336, name=qtp846514698-57336, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$27/644451172.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=57343, name=Scheduler-654441508, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=57341, name=qtp846514698-57341, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=57338, name=qtp846514698-57338, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=57340, name=qtp846514698-57340, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=57337, name=qtp846514698-57337-acceptor-0@3d5d18ec-ServerConnector@60323d6f{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:38927}, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=57342, name=qtp846514698-57342, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=57335, name=qtp846514698-57335, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$27/644451172.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=57458, name=Scheduler-910903452, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([422049B263044CDA]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=57336, name=qtp846514698-57336, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$27/644451172.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=57335, name=qtp846514698-57335, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$27/644451172.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=57343, name=Scheduler-654441508, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=57458, name=Scheduler-910903452, state=WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=57336, name=qtp846514698-57336, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$27/644451172.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=57335, name=qtp846514698-57335, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$27/644451172.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=57343, name=Scheduler-654441508, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=57458, name=Scheduler-910903452, state=WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([422049B263044CDA]:0)
FAILED: org.apache.solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest.testSimple
Error Message:
Waiting for collection testSimple2 Timeout waiting to see state for collection=testSimple2 :DocCollection(testSimple2//collections/testSimple2/state.json/26)={ "pullReplicas":"0", "replicationFactor":"2", "shards":{ "shard1":{ "range":"80000000-ffffffff", "state":"active", "replicas":{ "core_node2":{ "dataDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node2/data/", "base_url":"http://127.0.0.1:34471/solr", "node_name":"127.0.0.1:34471_solr", "type":"NRT", "force_set_state":"false", "ulogDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node2/data/tlog", "core":"testSimple2_shard1_replica_n1", "shared_storage":"true", "state":"down"}, "core_node5":{ "dataDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node5/data/", "base_url":"http://127.0.0.1:43695/solr", "node_name":"127.0.0.1:43695_solr", "type":"NRT", "force_set_state":"false", "ulogDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node5/data/tlog", "core":"testSimple2_shard1_replica_n3", "shared_storage":"true", "state":"active", "leader":"true"}}}, "shard2":{ "range":"0-7fffffff", "state":"active", "replicas":{ "core_node7":{ "dataDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node7/data/", "base_url":"http://127.0.0.1:34471/solr", "node_name":"127.0.0.1:34471_solr", "type":"NRT", "force_set_state":"false", "ulogDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node7/data/tlog", "core":"testSimple2_shard2_replica_n4", "shared_storage":"true", "state":"down"}, "core_node8":{ "dataDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node8/data/", "base_url":"http://127.0.0.1:43695/solr", "node_name":"127.0.0.1:43695_solr", "type":"NRT", "force_set_state":"false", "ulogDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node8/data/tlog", "core":"testSimple2_shard2_replica_n6", "shared_storage":"true", "state":"active", "leader":"true"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"2", "autoAddReplicas":"true", "nrtReplicas":"2", "tlogReplicas":"0"} Live Nodes: [127.0.0.1:37688_solr, 127.0.0.1:43695_solr] Last available state: DocCollection(testSimple2//collections/testSimple2/state.json/26)={ "pullReplicas":"0", "replicationFactor":"2", "shards":{ "shard1":{ "range":"80000000-ffffffff", "state":"active", "replicas":{ "core_node2":{ "dataDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node2/data/", "base_url":"http://127.0.0.1:34471/solr", "node_name":"127.0.0.1:34471_solr", "type":"NRT", "force_set_state":"false", "ulogDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node2/data/tlog", "core":"testSimple2_shard1_replica_n1", "shared_storage":"true", "state":"down"}, "core_node5":{ "dataDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node5/data/", "base_url":"http://127.0.0.1:43695/solr", "node_name":"127.0.0.1:43695_solr", "type":"NRT", "force_set_state":"false", "ulogDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node5/data/tlog", "core":"testSimple2_shard1_replica_n3", "shared_storage":"true", "state":"active", "leader":"true"}}}, "shard2":{ "range":"0-7fffffff", "state":"active", "replicas":{ "core_node7":{ "dataDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node7/data/", "base_url":"http://127.0.0.1:34471/solr", "node_name":"127.0.0.1:34471_solr", "type":"NRT", "force_set_state":"false", "ulogDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node7/data/tlog", "core":"testSimple2_shard2_replica_n4", "shared_storage":"true", "state":"down"}, "core_node8":{ "dataDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node8/data/", "base_url":"http://127.0.0.1:43695/solr", "node_name":"127.0.0.1:43695_solr", "type":"NRT", "force_set_state":"false", "ulogDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node8/data/tlog", "core":"testSimple2_shard2_replica_n6", "shared_storage":"true", "state":"active", "leader":"true"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"2", "autoAddReplicas":"true", "nrtReplicas":"2", "tlogReplicas":"0"}
Stack Trace:
java.lang.AssertionError: Waiting for collection testSimple2
Timeout waiting to see state for collection=testSimple2 :DocCollection(testSimple2//collections/testSimple2/state.json/26)={
"pullReplicas":"0",
"replicationFactor":"2",
"shards":{
"shard1":{
"range":"80000000-ffffffff",
"state":"active",
"replicas":{
"core_node2":{
"dataDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node2/data/",
"base_url":"http://127.0.0.1:34471/solr",
"node_name":"127.0.0.1:34471_solr",
"type":"NRT",
"force_set_state":"false",
"ulogDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node2/data/tlog",
"core":"testSimple2_shard1_replica_n1",
"shared_storage":"true",
"state":"down"},
"core_node5":{
"dataDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node5/data/",
"base_url":"http://127.0.0.1:43695/solr",
"node_name":"127.0.0.1:43695_solr",
"type":"NRT",
"force_set_state":"false",
"ulogDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node5/data/tlog",
"core":"testSimple2_shard1_replica_n3",
"shared_storage":"true",
"state":"active",
"leader":"true"}}},
"shard2":{
"range":"0-7fffffff",
"state":"active",
"replicas":{
"core_node7":{
"dataDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node7/data/",
"base_url":"http://127.0.0.1:34471/solr",
"node_name":"127.0.0.1:34471_solr",
"type":"NRT",
"force_set_state":"false",
"ulogDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node7/data/tlog",
"core":"testSimple2_shard2_replica_n4",
"shared_storage":"true",
"state":"down"},
"core_node8":{
"dataDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node8/data/",
"base_url":"http://127.0.0.1:43695/solr",
"node_name":"127.0.0.1:43695_solr",
"type":"NRT",
"force_set_state":"false",
"ulogDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node8/data/tlog",
"core":"testSimple2_shard2_replica_n6",
"shared_storage":"true",
"state":"active",
"leader":"true"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"2",
"autoAddReplicas":"true",
"nrtReplicas":"2",
"tlogReplicas":"0"}
Live Nodes: [127.0.0.1:37688_solr, 127.0.0.1:43695_solr]
Last available state: DocCollection(testSimple2//collections/testSimple2/state.json/26)={
"pullReplicas":"0",
"replicationFactor":"2",
"shards":{
"shard1":{
"range":"80000000-ffffffff",
"state":"active",
"replicas":{
"core_node2":{
"dataDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node2/data/",
"base_url":"http://127.0.0.1:34471/solr",
"node_name":"127.0.0.1:34471_solr",
"type":"NRT",
"force_set_state":"false",
"ulogDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node2/data/tlog",
"core":"testSimple2_shard1_replica_n1",
"shared_storage":"true",
"state":"down"},
"core_node5":{
"dataDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node5/data/",
"base_url":"http://127.0.0.1:43695/solr",
"node_name":"127.0.0.1:43695_solr",
"type":"NRT",
"force_set_state":"false",
"ulogDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node5/data/tlog",
"core":"testSimple2_shard1_replica_n3",
"shared_storage":"true",
"state":"active",
"leader":"true"}}},
"shard2":{
"range":"0-7fffffff",
"state":"active",
"replicas":{
"core_node7":{
"dataDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node7/data/",
"base_url":"http://127.0.0.1:34471/solr",
"node_name":"127.0.0.1:34471_solr",
"type":"NRT",
"force_set_state":"false",
"ulogDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node7/data/tlog",
"core":"testSimple2_shard2_replica_n4",
"shared_storage":"true",
"state":"down"},
"core_node8":{
"dataDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node8/data/",
"base_url":"http://127.0.0.1:43695/solr",
"node_name":"127.0.0.1:43695_solr",
"type":"NRT",
"force_set_state":"false",
"ulogDir":"hdfs://localhost:38425/solr_hdfs_home/testSimple2/core_node8/data/tlog",
"core":"testSimple2_shard2_replica_n6",
"shared_storage":"true",
"state":"active",
"leader":"true"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"2",
"autoAddReplicas":"true",
"nrtReplicas":"2",
"tlogReplicas":"0"}
at __randomizedtesting.SeedInfo.seed([422049B263044CDA:7A936D4C44F7980B]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:310)
at org.apache.solr.cloud.autoscaling.AutoAddReplicasIntegrationTest.testSimple(AutoAddReplicasIntegrationTest.java:169)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 14936 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest_422049B263044CDA-001/init-core-data-001
[junit4] 2> 3281048 WARN (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=8 numCloses=8
[junit4] 2> 3281048 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 3281051 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 3281051 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 3281093 WARN (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 3281115 WARN (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3281117 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 3281118 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3281118 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3281118 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 3281119 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@291e729{static,/static,jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
[junit4] 2> 3281320 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@5f3ffb83{hdfs,/,file:///x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/jetty-localhost-42071-hdfs-_-any-977804923630617668.dir/webapp/,AVAILABLE}{/hdfs}
[junit4] 2> 3281321 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4cfe46c4{HTTP/1.1,[http/1.1]}{localhost:42071}
[junit4] 2> 3281321 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.e.j.s.Server Started @3281428ms
[junit4] 2> 3281471 WARN (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3281472 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 3281473 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3281473 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3281473 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 3281473 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@45c6de5a{static,/static,jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
[junit4] 2> 3281631 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@f87f7c3{datanode,/,file:///x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/jetty-localhost-43337-datanode-_-any-6301432980286235963.dir/webapp/,AVAILABLE}{/datanode}
[junit4] 2> 3281631 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2a74309e{HTTP/1.1,[http/1.1]}{localhost:43337}
[junit4] 2> 3281631 INFO (SUITE-HdfsAutoAddReplicasIntegrationTest-seed#[422049B263044CDA]-worker) [ ] o.e.j.s.Server Started @3281738ms
[junit4] 2> 3281810 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x3eb0f6e0f00dc5f: Processing first storage report for DS-76344bdc-793c-4165-bbe8-57255ae0d774 from datanode ebc27246-0d0e-4b41-82c0-1cb3cccbdcef
[junit4] 2> 3281810 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x3eb0f6e0f00dc5f: from storage DS-76344bdc-793c-4165-bbe8-57255ae0d774 node DatanodeRegistration(127.0.0.1:45303, datanodeUuid=ebc27246-0d0e-4b41-82c0-1cb3cccbdcef, infoPort=36074, infoSecurePort=0, ipcPort=43773, storageInfo=lv=-57;cid=testClusterID;nsid=676239334;c=1553468903253), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
[junit4] 2> 3281810 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x3eb0f6e0f00dc5f: Processing first storage report for DS-4384ba6e-a523-4bda-9852-8dfaf0a3279c from datanode ebc27246-0d0e-4b41-82c0-1cb3cccbdcef
[junit4] 2> 3281810 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x3eb0f6e0f00dc5f: from storage DS-4384ba6e-a523-4bda-9852-8dfaf0a3279c node DatanodeRegistration(127.0.0.1:45303, datanodeUuid=ebc27246-0d0e-4b41-82c0-1cb3cccbdcef, infoPort=36074, infoSecurePort=0, ipcPort=43773, storageInfo=lv=-57;cid=testClusterID;nsid=676239334;c=1553468903253), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
[junit4] 2> 3282046 INFO (TEST-HdfsAutoAddReplicasIntegrationTest.testSimple-seed#[422049B263044CDA]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSimple
[junit4] 2> 3282047 INFO (TEST-HdfsAutoAddReplicasIntegrationTest.testSimple-seed#[422049B263044CDA]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest_422049B263044CDA-001/tempDir-002
[junit4] 2> 3282047 INFO (TEST-HdfsAutoAddReplicasIntegrationTest.testSimple-seed#[422049B263044CDA]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3282048 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3282048 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3282148 INFO (TEST-HdfsAutoAddReplicasIntegrationTest.testSimple-seed#[422049B263044CDA]) [ ] o.a.s.c.ZkTestServer start zk server on port:45090
[junit4] 2> 3282148 INFO (TEST-HdfsAutoAddReplicasIntegrationTest.testSimple-seed#[422049B263044CDA]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:45090
[junit4] 2> 3282148 INFO (TEST-HdfsAutoAddReplicasIntegrationTest.testSimple-seed#[422049B263044CDA]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 45090
[junit4] 2> 3282158 INFO (zkConnectionManagerCallback-15863-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3282161 INFO (zkConnectionManagerCallback-15865-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3282162 INFO (zkConnectionManagerCallback-15867-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3282164 WARN (jetty-launcher-15868-thread-1) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 3282164 WARN (jetty-launcher-15868-thread-2) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 3282165 INFO (jetty-launcher-15868-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 3282165 INFO (jetty-launcher-15868-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 3282165 INFO (jetty-launcher-15868-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 3282165 INFO (jetty-launcher-15868-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 3282165 INFO (jetty-launcher-15868-thread-1) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 3282165 INFO (jetty-launcher-15868-thread-2) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 3282165 WARN (jetty-launcher-15868-thread-3) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 3282165 INFO (jetty-launcher-15868-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 3282165 INFO (jetty-launcher-15868-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 3282165 INFO (jetty-launcher-15868-thread-3) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 3282169 INFO (jetty-launcher-15868-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3282169 INFO (jetty-launcher-15868-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3282169 INFO (jetty-launcher-15868-thread-1) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 3282169 INFO (jetty-launcher-15868-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@56c6e756{/solr,null,AVAILABLE}
[junit4] 2> 3282169 INFO (jetty-launcher-15868-thread-3) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3282169 INFO (jetty-launcher-15868-thread-3) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3282170 INFO (jetty-launcher-15868-thread-3) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 3282170 INFO (jetty-launcher-15868-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@24ff6cb2{/solr,null,AVAILABLE}
[junit4] 2> 3282170 INFO (jetty-launcher-15868-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3282170 INFO (jetty-launcher-15868-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3282170 INFO (jetty-launcher-15868-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@d4ef162{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:37688}
[junit4] 2> 3282170 INFO (jetty-launcher-15868-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1d516dfd{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:34471}
[junit4] 2> 3282170 INFO (jetty-launcher-15868-thread-1) [ ] o.e.j.s.Server Started @3282278ms
[junit4] 2> 3282170 INFO (jetty-launcher-15868-thread-3) [ ] o.e.j.s.Server Started @3282278ms
[junit4] 2> 3282170 INFO (jetty-launcher-15868-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37688}
[junit4] 2> 3282170 INFO (jetty-launcher-15868-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34471}
[junit4] 2> 3282171 ERROR (jetty-launcher-15868-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3282171 ERROR (jetty-launcher-15868-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3282171 INFO (jetty-launcher-15868-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 3282171 INFO (jetty-launcher-15868-thread-3) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 3282171 INFO (jetty-launcher-15868-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 3282171 INFO (jetty-launcher-15868-thread-2) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 3282171 INFO (jetty-launcher-15868-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3282171 INFO (jetty-launcher-15868-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3282171 INFO (jetty-launcher-15868-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-03-24T23:08:24.356Z
[junit4] 2> 3282171 INFO (jetty-launcher-15868-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 3282171 INFO (jetty-launcher-15868-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3282171 INFO (jetty-launcher-15868-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3282172 INFO (jetty-launcher-15868-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-03-24T23:08:24.357Z
[junit4] 2> 3282228 INFO (jetty-launcher-15868-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@38f1f80d{/solr,null,AVAILABLE}
[junit4] 2> 3282229 INFO (jetty-launcher-15868-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5aefefdd{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:43695}
[junit4] 2> 3282229 INFO (jetty-launcher-15868-thread-2) [ ] o.e.j.s.Server Started @3282336ms
[junit4] 2> 3282229 INFO (jetty-launcher-15868-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43695}
[junit4] 2> 3282229 ERROR (jetty-launcher-15868-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3282229 INFO (jetty-launcher-15868-thread-2) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 3282229 INFO (jetty-launcher-15868-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 3282230 INFO (jetty-launcher-15868-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3282230 INFO (jetty-launcher-15868-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3282230 INFO (jetty-launcher-15868-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-03-24T23:08:24.415Z
[junit4] 2> 3282232 INFO (zkConnectionManagerCallback-15870-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3282232 INFO (zkConnectionManagerCallback-15872-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3282237 INFO (jetty-launcher-15868-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3282237 INFO (jetty-launcher-15868-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3282243 INFO (jetty-launcher-15868-thread-3) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3282243 INFO (jetty-launcher-15868-thread-3) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3282243 INFO (jetty-launcher-15868-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3282243 INFO (jetty-launcher-15868-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3282244 INFO (jetty-launcher-15868-thread-3) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4a5cb000, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3282245 INFO (zkConnectionManagerCallback-15874-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3282245 INFO (jetty-launcher-15868-thread-1) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4a5cb000, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3282245 INFO (jetty-launcher-15868-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3282249 INFO (jetty-launcher-15868-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3282249 INFO (jetty-launcher-15868-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3282251 INFO (jetty-launcher-15868-thread-2) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4a5cb000, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3282299 INFO (jetty-launcher-15868-thread-3) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 3282301 WARN (jetty-launcher-15868-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@25b503b4[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3282305 WARN (jetty-launcher-15868-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@3563b676[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3282306 INFO (jetty-launcher-15868-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45090/solr
[junit4] 2> 3282308 INFO (zkConnectionManagerCallback-15883-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3282310 INFO (zkConnectionManagerCallback-15885-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3282370 INFO (jetty-launcher-15868-thread-1) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 3282372 WARN (jetty-launcher-15868-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@36daa332[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3282375 WARN (jetty-launcher-15868-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@1cec3e30[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3282376 INFO (jetty-launcher-15868-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45090/solr
[junit4] 2> 3282377 INFO (zkConnectionManagerCallback-15893-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3282380 INFO (zkConnectionManagerCallback-15895-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3282386 INFO (jetty-launcher-15868-thread-3) [n:127.0.0.1:37688_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37688_solr
[junit4] 2> 3282387 INFO (jetty-launcher-15868-thread-3) [n:127.0.0.1:37688_solr ] o.a.s.c.Overseer Overseer (id=73014659461611527-127.0.0.1:37688_solr-n_0000000000) starting
[junit4] 2> 3282390 INFO (jetty-launcher-15868-thread-1) [n:127.0.0.1:34471_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34471_solr
[junit4] 2> 3282392 INFO (zkCallback-15884-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3282392 INFO (zkCallback-15894-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3282395 INFO (zkConnectionManagerCallback-15902-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3282396 INFO (jetty-launcher-15868-thread-3) [n:127.0.0.1:37688_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3282397 INFO (jetty-launcher-15868-thread-3) [n:127.0.0.1:37688_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45090/solr ready
[junit4] 2> 3282398 INFO (OverseerStateUpdate-73014659461611527-127.0.0.1:37688_solr-n_0000000000) [n:127.0.0.1:37688_solr ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:37688_solr
[junit4] 2> 3282399 INFO (jetty-launcher-15868-thread-3) [n:127.0.0.1:37688_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:37688_solr as DOWN
[junit4] 2> 3282400 INFO (jetty-launcher-15868-thread-3) [n:127.0.0.1:37688_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3282400 INFO (jetty-launcher-15868-thread-3) [n:127.0.0.1:37688_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37688_solr
[junit4] 2> 3282402 INFO (zkCallback-15894-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3282402 INFO (zkCallback-15901-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3282403 INFO (zkCallback-15884-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3282421 INFO (zkConnectionManagerCallback-15907-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3282422 INFO (jetty-launcher-15868-thread-1) [n:127.0.0.1:34471_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3282424 INFO (jetty-launcher-15868-thread-1) [n:127.0.0.1:34471_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45090/solr ready
[junit4] 2> 3282424 INFO (jetty-launcher-15868-thread-1) [n:127.0.0.1:34471_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 3282426 INFO (jetty-launcher-15868-thread-3) [n:127.0.0.1:37688_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 3282452 INFO (jetty-launcher-15868-thread-3) [n:127.0.0.1:37688_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a5cb000
[junit4] 2> 3282455 INFO (jetty-launcher-15868-thread-1) [n:127.0.0.1:34471_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a5cb000
[junit4] 2> 3282479 INFO (jetty-launcher-15868-thread-3) [n:127.0.0.1:37688_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a5cb000
[junit4] 2> 3282479 INFO (jetty-launcher-15868-thread-3) [n:127.0.0.1:37688_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a5cb000
[junit4] 2> 3282480 INFO (jetty-launcher-15868-thread-3) [n:127.0.0.1:37688_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest_422049B263044CDA-001/tempDir-002/node3/.
[junit4] 2> 3282483 INFO (jetty-launcher-15868-thread-1) [n:127.0.0.1:34471_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a5cb000
[junit4] 2> 3282484 INFO (jetty-launcher-15868-thread-1) [n:127.0.0.1:34471_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a5cb000
[junit4] 2> 3282486 INFO (jetty-launcher-15868-thread-1) [n:127.0.0.1:34471_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest_422049B263044CDA-001/tempDir-002/node1/.
[junit4] 2> 3282525 INFO (jetty-launcher-15868-thread-2) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 3282527 WARN (jetty-launcher-15868-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@6ed4d1fa[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3282531 WARN (jetty-launcher-15868-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@553115d1[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3282532 INFO (jetty-launcher-15868-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45090/solr
[junit4] 2> 3282534 INFO (zkConnectionManagerCallback-15915-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3282536 INFO (zkConnectionManagerCallback-15917-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3282541 INFO (jetty-launcher-15868-thread-2) [n:127.0.0.1:43695_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3282545 INFO (jetty-launcher-15868-thread-2) [n:127.0.0.1:43695_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:43695_solr as DOWN
[junit4] 2> 3282546 INFO (jetty-launcher-15868-thread-2) [n:127.0.0.1:43695_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3282546 INFO (jetty-launcher-15868-thread-2) [n:127.0.0.1:43695_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43695_solr
[junit4] 2> 3282547 INFO (zkCallback-15901-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3282548 INFO (zkCallback-15894-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3282548 INFO (zkCallback-15884-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3282550 INFO (zkCallback-15906-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3282550 INFO (zkCallback-15916-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3282575 INFO (zkConnectionManagerCallback-15924-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3282577 INFO (jetty-launcher-15868-thread-2) [n:127.0.0.1:43695_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 3282578 INFO (jetty-launcher-15868-thread-2) [n:127.0.0.1:43695_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45090/solr ready
[junit4] 2> 3282578 INFO (jetty-launcher-15868-thread-2) [n:127.0.0.1:43695_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 3282606 INFO (jetty-launcher-15868-thread-2) [n:127.0.0.1:43695_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a5cb000
[junit4] 2> 3282623 INFO (jetty-launcher-15868-thread-2) [n:127.0.0.1:43695_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a5cb000
[junit4] 2> 3282623 INFO (jetty-launcher-15868-thread-2) [n:127.0.0.1:43695_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a5cb000
[junit4] 2> 3282625 INFO (jetty-launcher-15868-thread-2) [n:127.0.0.1:43695_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest_422049B263044CDA-001/tempDir-002/node2/.
[junit4] 2> 3282681 INFO (zkConnectionManagerCallback-15930-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3282683 INFO (TEST-HdfsAutoAddReplicasIntegrationTest.testSimple-seed#[422049B263044CDA]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 3282684 INFO (TEST-HdfsAutoAddReplicasIntegrationTest.testSimple-seed#[422049B263044CDA]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45090/solr ready
[junit4] 2> 3282690 INFO (qtp1688262693-52812) [n:127.0.0.1:43695_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster params={wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 3282692 INFO (qtp1424580020-52813) [n:127.0.0.1:34471_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf&maxShardsPerNode=2&autoAddReplicas=true&name=testSimple1&nrtReplicas=2&action=CREATE&numShards=2&createNodeSet=127.0.0.1:37688_solr,127.0.0.1:34471_solr&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3282695 INFO (OverseerThreadFactory-14682-thread-1-processing-n:127.0.0.1:37688_solr) [n:127.0.0.1:37688_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection testSimple1
[junit4] 2> 3282812 INFO (OverseerStateUpdate-73014659461611527-127.0.0.1:37688_solr-n_0000000000) [n:127.0.0.1:37688_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testSimple1",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"testSimple1_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:37688/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 3282816 INFO (OverseerStateUpdate-73014659461611527-127.0.0.1:37688_solr-n_0000000000) [n:127.0.0.1:37688_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testSimple1",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"testSimple1_shard1_replica_n3",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:34471/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 3282818 INFO (OverseerStateUpdate-73014659461611527-127.0.0.1:37688_solr-n_0000000000) [n:127.0.0.1:37688_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testSimple1",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"testSimple1_shard2_replica_n4",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:37688/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 3282821 INFO (OverseerStateUpdate-73014659461611527-127.0.0.1:37688_solr-n_0000000000) [n:127.0.0.1:37688_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testSimple1",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"testSimple1_shard2_replica_n6",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:34471/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 3283029 INFO (qtp1424580020-52817) [n:127.0.0.1:34471_solr x:testSimple1_shard1_replica_n3] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=testSimple1_shard1_replica_n3&action=CREATE&numShards=2&collection=testSimple1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3283029 INFO (qtp1424580020-52810) [n:127.0.0.1:34471_solr x:testSimple1_shard2_replica_n6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf&newCollection=true&name=testSimple1_shard2_replica_n6&action=CREATE&numShards=2&collection=testSimple1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3283030 INFO (qtp1424580020-52810) [n:127.0.0.1:34471_solr x:testSimple1_shard2_replica_n6] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3283030 INFO (qtp1493415877-52821) [n:127.0.0.1:37688_solr x:testSimple1_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=testSimple1_shard1_replica_n1&action=CREATE&numShards=2&collection=testSimple1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3283031 INFO (qtp1493415877-52811) [n:127.0.0.1:37688_solr x:testSimple1_shard2_replica_n4] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=testSimple1_shard2_replica_n4&action=CREATE&numShards=2&collection=testSimple1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3284050 INFO (qtp1493415877-52821) [n:127.0.0.1:37688_solr c:testSimple1 s:shard1 r:core_node2 x:testSimple1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 3284050 INFO (qtp1424580020-52817) [n:127.0.0.1:34471_solr c:testSimple1 s:shard1 r:core_node5 x:testSimple1_shard1_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 3284069 INFO (qtp1424580020-52810) [n:127.0.0.1:34471_solr c:testSimple1 s:shard2 r:core_node8 x:testSimple1_shard2_replica_n6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 3284070 INFO (qtp1493415877-52811) [n:127.0.0.1:37688_solr c:testSimple1 s:shard2 r:core_node7 x:testSimple1_shard2_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 3284082 INFO (qtp1424580020-52810) [n:127.0.0.1:34471_solr c:testSimple1 s:shard2 r:core_node8 x:testSimple1_shard2_replica_n6] o.a.s.s.IndexSchema [testSimple1_shard2_replica_n6] Schema name=minimal
[junit4] 2> 3284084 INFO (qtp1493415877-52811) [n:127.0.0.1:37688_solr c:testSimple1 s:shard2 r:core_node7 x:testSimple1_shard2_replica_n4] o.a.s.s.IndexSchema [testSimple1_shard2_replica_n4] Schema name=minimal
[junit4] 2> 3284086 INFO (qtp1424580020-52810) [n:127.0.0.1:34471_solr c:testSimple1 s:shard2 r:core_node8 x:testSimple1_shard2_replica_n6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3284086 INFO (qtp1424580020-52810) [n:127.0.0.1:34471_solr c:testSimple1 s:shard2 r:core_node8 x:testSimple1_shard2_replica_n6] o.a.s.c.CoreContainer Creating SolrCore 'testSimple1_shard2_replica_n6' using configuration from collection testSimple1, trusted=true
[junit4] 2> 3284086 INFO (qtp1424580020-52810) [n:127.0.0.1:34471_solr c:testSimple1 s:shard2 r:core_node8 x:testSimple1_shard2_replica_n6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.testSimple1.shard2.replica_n6' (registry 'solr.core.testSimple1.shard2.replica_n6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a5cb000
[junit4] 2> 3284086 INFO (qtp1424580020-52810) [n:127.0.0.1:34471_solr c:testSimple1 s:shard2 r:core_node8 x:testSimple1_shard2_replica_n6] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:38425/solr_hdfs_home
[junit4] 2> 3284086 INFO (qtp1424580020-52810) [n:127.0.0.1:34471_solr c:testSimple1 s:shard2 r:core_node8 x:testSimple1_shard2_replica_n6] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 3284086 INFO (qtp1424580020-52810) [n:127.0.0.1:34471_solr c:testSimple1 s:shard2 r:core_node8 x:testSimple1_shard2_replica_n6] o.a.s.c.SolrCore [[testSimple1_shard2_replica_n6] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest_422049B263044CDA-001/tempDir-002/node1/testSimple1_shard2_replica_n6], dataDir=[hdfs://localhost:38425/solr_hdfs_home/testSimple1/core_node8/data/]
[junit4] 2> 3284088 INFO (qtp1493415877-52821) [n:127.0.0.1:37688_solr c:testSimple1 s:shard1 r:core_node2 x:testSimple1_shard1_replica_n1] o.a.s.s.IndexSchema [testSimple1_shard1_replica_n1] Schema name=minimal
[junit4] 2> 3284089 INFO (qtp1424580020-52810) [n:127.0.0.1:34471_solr c:testSimple1 s:shard2 r:core_node8 x:testSimple1_shard2_replica_n6] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:38425/solr_hdfs_home/testSimple1/core_node8/data/snapshot_metadata
[junit4] 2> 3284092 INFO (qtp1424580020-52817) [n:127.0.0.1:34471_solr c:testSimple1 s:shard1 r:core_node5 x:testSimple1_shard1_replica_n3] o.a.s.s.IndexSchema [testSimple1_shard1_replica_n3] Schema name=minimal
[junit4] 2> 3284116 INFO (qtp1493415877-52811) [n:127.0.0.1:37688_solr c:testSimple1 s:shard2 r:core_node7 x:testSimple1_shard2_replica_n4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3284116 INFO (qtp1493415877-52811) [n:127.0.0.1:37688_solr c:testSimple1 s:shard2 r:core_node7 x:testSimple1_shard2_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'testSimple1_shard2_replica_n4' using configuration from collection testSimple1, trusted=true
[junit4] 2> 3284116 INFO (qtp1493415877-52811) [n:127.0.0.1:37688_solr c:testSimple1 s:shard2 r:core_node7 x:testSimple1_shard2_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.testSimple1.shard2.replica_n4' (registry 'solr.core.testSimple1.shard2.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a5cb000
[junit4] 2> 3284116 INFO (qtp1493415877-52811) [n:127.0.0.1:37688_solr c:testSimple1 s:shard2 r:core_node7 x:testSimple1_shard2_replica_n4] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:38425/solr_hdfs_home
[junit4] 2> 3284117 INFO (qtp1493415877-52811) [n:127.0.0.1:37688_solr c:testSimple1 s:shard2 r:core_node7 x:testSimple1_shard2_replica_n4] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 3284117 INFO (qtp1493415877-52811) [n:127.0.0.1:37688_solr c:testSimple1 s:shard2 r:core_node7 x:testSimple1_shard2_replica_n4] o.a.s.c.SolrCore [[testSimple1_shard2_replica_n4] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest_422
[...truncated too long message...]
= /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null1344801139
[copy] Copying 240 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null1344801139
[delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null1344801139
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 61ms :: artifacts dl 4ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 323 minutes 54 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2847)
at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2726)
at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2707)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3086)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1072)
at hudson.FilePath.act(FilePath.java:1061)
at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1835)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3088)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1074)
at hudson.FilePath.act(FilePath.java:1061)
at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1835)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
[JENKINS] Lucene-Solr-BadApples-NightlyTests-master - Build # 54 -
Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-master/54/
6 tests failed.
FAILED: org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.testCollectionsAPI
Error Message:
Failed while waiting for active collection Timeout waiting to see state for collection=awhollynewcollection_0 :DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/5)={ "pullReplicas":"0", "replicationFactor":"4", "shards":{ "shard1":{ "range":"80000000-b332ffff", "state":"active", "replicas":{ "core_node2":{ "core":"awhollynewcollection_0_shard1_replica_n1", "base_url":"http://127.0.0.1:34042/solr", "node_name":"127.0.0.1:34042_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node4":{ "core":"awhollynewcollection_0_shard1_replica_n3", "base_url":"http://127.0.0.1:44384/solr", "node_name":"127.0.0.1:44384_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node6":{ "core":"awhollynewcollection_0_shard1_replica_n5", "base_url":"http://127.0.0.1:43110/solr", "node_name":"127.0.0.1:43110_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node9":{ "core":"awhollynewcollection_0_shard1_replica_n7", "base_url":"http://127.0.0.1:35026/solr", "node_name":"127.0.0.1:35026_solr", "state":"down", "type":"NRT", "force_set_state":"false"}}}, "shard2":{ "range":"b3330000-e665ffff", "state":"active", "replicas":{ "core_node11":{ "core":"awhollynewcollection_0_shard2_replica_n8", "base_url":"http://127.0.0.1:34042/solr", "node_name":"127.0.0.1:34042_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node13":{ "core":"awhollynewcollection_0_shard2_replica_n10", "base_url":"http://127.0.0.1:44384/solr", "node_name":"127.0.0.1:44384_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node15":{ "core":"awhollynewcollection_0_shard2_replica_n12", "base_url":"http://127.0.0.1:43110/solr", "node_name":"127.0.0.1:43110_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node17":{ "core":"awhollynewcollection_0_shard2_replica_n14", "base_url":"http://127.0.0.1:35026/solr", "node_name":"127.0.0.1:35026_solr", "state":"down", "type":"NRT", "force_set_state":"false"}}}, "shard3":{ "range":"e6660000-1998ffff", "state":"active", "replicas":{ "core_node19":{ "core":"awhollynewcollection_0_shard3_replica_n16", "base_url":"http://127.0.0.1:34042/solr", "node_name":"127.0.0.1:34042_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node21":{ "core":"awhollynewcollection_0_shard3_replica_n18", "base_url":"http://127.0.0.1:44384/solr", "node_name":"127.0.0.1:44384_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node23":{ "core":"awhollynewcollection_0_shard3_replica_n20", "base_url":"http://127.0.0.1:43110/solr", "node_name":"127.0.0.1:43110_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node25":{ "core":"awhollynewcollection_0_shard3_replica_n22", "base_url":"http://127.0.0.1:35026/solr", "node_name":"127.0.0.1:35026_solr", "state":"down", "type":"NRT", "force_set_state":"false"}}}, "shard4":{ "range":"19990000-4ccbffff", "state":"active", "replicas":{ "core_node27":{ "core":"awhollynewcollection_0_shard4_replica_n24", "base_url":"http://127.0.0.1:34042/solr", "node_name":"127.0.0.1:34042_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node29":{ "core":"awhollynewcollection_0_shard4_replica_n26", "base_url":"http://127.0.0.1:44384/solr", "node_name":"127.0.0.1:44384_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node31":{ "core":"awhollynewcollection_0_shard4_replica_n28", "base_url":"http://127.0.0.1:43110/solr", "node_name":"127.0.0.1:43110_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node33":{ "core":"awhollynewcollection_0_shard4_replica_n30", "base_url":"http://127.0.0.1:35026/solr", "node_name":"127.0.0.1:35026_solr", "state":"down", "type":"NRT", "force_set_state":"false"}}}, "shard5":{ "range":"4ccc0000-7fffffff", "state":"active", "replicas":{ "core_node35":{ "core":"awhollynewcollection_0_shard5_replica_n32", "base_url":"http://127.0.0.1:34042/solr", "node_name":"127.0.0.1:34042_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node37":{ "core":"awhollynewcollection_0_shard5_replica_n34", "base_url":"http://127.0.0.1:44384/solr", "node_name":"127.0.0.1:44384_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node39":{ "core":"awhollynewcollection_0_shard5_replica_n36", "base_url":"http://127.0.0.1:43110/solr", "node_name":"127.0.0.1:43110_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node40":{ "core":"awhollynewcollection_0_shard5_replica_n38", "base_url":"http://127.0.0.1:35026/solr", "node_name":"127.0.0.1:35026_solr", "state":"down", "type":"NRT", "force_set_state":"false"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"6", "autoAddReplicas":"false", "nrtReplicas":"4", "tlogReplicas":"0"} Live Nodes: [127.0.0.1:34042_solr, 127.0.0.1:35026_solr, 127.0.0.1:43110_solr, 127.0.0.1:44384_solr] Last available state: DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/5)={ "pullReplicas":"0", "replicationFactor":"4", "shards":{ "shard1":{ "range":"80000000-b332ffff", "state":"active", "replicas":{ "core_node2":{ "core":"awhollynewcollection_0_shard1_replica_n1", "base_url":"http://127.0.0.1:34042/solr", "node_name":"127.0.0.1:34042_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node4":{ "core":"awhollynewcollection_0_shard1_replica_n3", "base_url":"http://127.0.0.1:44384/solr", "node_name":"127.0.0.1:44384_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node6":{ "core":"awhollynewcollection_0_shard1_replica_n5", "base_url":"http://127.0.0.1:43110/solr", "node_name":"127.0.0.1:43110_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node9":{ "core":"awhollynewcollection_0_shard1_replica_n7", "base_url":"http://127.0.0.1:35026/solr", "node_name":"127.0.0.1:35026_solr", "state":"down", "type":"NRT", "force_set_state":"false"}}}, "shard2":{ "range":"b3330000-e665ffff", "state":"active", "replicas":{ "core_node11":{ "core":"awhollynewcollection_0_shard2_replica_n8", "base_url":"http://127.0.0.1:34042/solr", "node_name":"127.0.0.1:34042_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node13":{ "core":"awhollynewcollection_0_shard2_replica_n10", "base_url":"http://127.0.0.1:44384/solr", "node_name":"127.0.0.1:44384_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node15":{ "core":"awhollynewcollection_0_shard2_replica_n12", "base_url":"http://127.0.0.1:43110/solr", "node_name":"127.0.0.1:43110_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node17":{ "core":"awhollynewcollection_0_shard2_replica_n14", "base_url":"http://127.0.0.1:35026/solr", "node_name":"127.0.0.1:35026_solr", "state":"down", "type":"NRT", "force_set_state":"false"}}}, "shard3":{ "range":"e6660000-1998ffff", "state":"active", "replicas":{ "core_node19":{ "core":"awhollynewcollection_0_shard3_replica_n16", "base_url":"http://127.0.0.1:34042/solr", "node_name":"127.0.0.1:34042_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node21":{ "core":"awhollynewcollection_0_shard3_replica_n18", "base_url":"http://127.0.0.1:44384/solr", "node_name":"127.0.0.1:44384_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node23":{ "core":"awhollynewcollection_0_shard3_replica_n20", "base_url":"http://127.0.0.1:43110/solr", "node_name":"127.0.0.1:43110_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node25":{ "core":"awhollynewcollection_0_shard3_replica_n22", "base_url":"http://127.0.0.1:35026/solr", "node_name":"127.0.0.1:35026_solr", "state":"down", "type":"NRT", "force_set_state":"false"}}}, "shard4":{ "range":"19990000-4ccbffff", "state":"active", "replicas":{ "core_node27":{ "core":"awhollynewcollection_0_shard4_replica_n24", "base_url":"http://127.0.0.1:34042/solr", "node_name":"127.0.0.1:34042_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node29":{ "core":"awhollynewcollection_0_shard4_replica_n26", "base_url":"http://127.0.0.1:44384/solr", "node_name":"127.0.0.1:44384_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node31":{ "core":"awhollynewcollection_0_shard4_replica_n28", "base_url":"http://127.0.0.1:43110/solr", "node_name":"127.0.0.1:43110_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node33":{ "core":"awhollynewcollection_0_shard4_replica_n30", "base_url":"http://127.0.0.1:35026/solr", "node_name":"127.0.0.1:35026_solr", "state":"down", "type":"NRT", "force_set_state":"false"}}}, "shard5":{ "range":"4ccc0000-7fffffff", "state":"active", "replicas":{ "core_node35":{ "core":"awhollynewcollection_0_shard5_replica_n32", "base_url":"http://127.0.0.1:34042/solr", "node_name":"127.0.0.1:34042_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node37":{ "core":"awhollynewcollection_0_shard5_replica_n34", "base_url":"http://127.0.0.1:44384/solr", "node_name":"127.0.0.1:44384_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node39":{ "core":"awhollynewcollection_0_shard5_replica_n36", "base_url":"http://127.0.0.1:43110/solr", "node_name":"127.0.0.1:43110_solr", "state":"down", "type":"NRT", "force_set_state":"false"}, "core_node40":{ "core":"awhollynewcollection_0_shard5_replica_n38", "base_url":"http://127.0.0.1:35026/solr", "node_name":"127.0.0.1:35026_solr", "state":"down", "type":"NRT", "force_set_state":"false"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"6", "autoAddReplicas":"false", "nrtReplicas":"4", "tlogReplicas":"0"}
Stack Trace:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=awhollynewcollection_0 :DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/5)={
"pullReplicas":"0",
"replicationFactor":"4",
"shards":{
"shard1":{
"range":"80000000-b332ffff",
"state":"active",
"replicas":{
"core_node2":{
"core":"awhollynewcollection_0_shard1_replica_n1",
"base_url":"http://127.0.0.1:34042/solr",
"node_name":"127.0.0.1:34042_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node4":{
"core":"awhollynewcollection_0_shard1_replica_n3",
"base_url":"http://127.0.0.1:44384/solr",
"node_name":"127.0.0.1:44384_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node6":{
"core":"awhollynewcollection_0_shard1_replica_n5",
"base_url":"http://127.0.0.1:43110/solr",
"node_name":"127.0.0.1:43110_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node9":{
"core":"awhollynewcollection_0_shard1_replica_n7",
"base_url":"http://127.0.0.1:35026/solr",
"node_name":"127.0.0.1:35026_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"}}},
"shard2":{
"range":"b3330000-e665ffff",
"state":"active",
"replicas":{
"core_node11":{
"core":"awhollynewcollection_0_shard2_replica_n8",
"base_url":"http://127.0.0.1:34042/solr",
"node_name":"127.0.0.1:34042_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node13":{
"core":"awhollynewcollection_0_shard2_replica_n10",
"base_url":"http://127.0.0.1:44384/solr",
"node_name":"127.0.0.1:44384_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node15":{
"core":"awhollynewcollection_0_shard2_replica_n12",
"base_url":"http://127.0.0.1:43110/solr",
"node_name":"127.0.0.1:43110_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node17":{
"core":"awhollynewcollection_0_shard2_replica_n14",
"base_url":"http://127.0.0.1:35026/solr",
"node_name":"127.0.0.1:35026_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"}}},
"shard3":{
"range":"e6660000-1998ffff",
"state":"active",
"replicas":{
"core_node19":{
"core":"awhollynewcollection_0_shard3_replica_n16",
"base_url":"http://127.0.0.1:34042/solr",
"node_name":"127.0.0.1:34042_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node21":{
"core":"awhollynewcollection_0_shard3_replica_n18",
"base_url":"http://127.0.0.1:44384/solr",
"node_name":"127.0.0.1:44384_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node23":{
"core":"awhollynewcollection_0_shard3_replica_n20",
"base_url":"http://127.0.0.1:43110/solr",
"node_name":"127.0.0.1:43110_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node25":{
"core":"awhollynewcollection_0_shard3_replica_n22",
"base_url":"http://127.0.0.1:35026/solr",
"node_name":"127.0.0.1:35026_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"}}},
"shard4":{
"range":"19990000-4ccbffff",
"state":"active",
"replicas":{
"core_node27":{
"core":"awhollynewcollection_0_shard4_replica_n24",
"base_url":"http://127.0.0.1:34042/solr",
"node_name":"127.0.0.1:34042_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node29":{
"core":"awhollynewcollection_0_shard4_replica_n26",
"base_url":"http://127.0.0.1:44384/solr",
"node_name":"127.0.0.1:44384_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node31":{
"core":"awhollynewcollection_0_shard4_replica_n28",
"base_url":"http://127.0.0.1:43110/solr",
"node_name":"127.0.0.1:43110_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node33":{
"core":"awhollynewcollection_0_shard4_replica_n30",
"base_url":"http://127.0.0.1:35026/solr",
"node_name":"127.0.0.1:35026_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"}}},
"shard5":{
"range":"4ccc0000-7fffffff",
"state":"active",
"replicas":{
"core_node35":{
"core":"awhollynewcollection_0_shard5_replica_n32",
"base_url":"http://127.0.0.1:34042/solr",
"node_name":"127.0.0.1:34042_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node37":{
"core":"awhollynewcollection_0_shard5_replica_n34",
"base_url":"http://127.0.0.1:44384/solr",
"node_name":"127.0.0.1:44384_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node39":{
"core":"awhollynewcollection_0_shard5_replica_n36",
"base_url":"http://127.0.0.1:43110/solr",
"node_name":"127.0.0.1:43110_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node40":{
"core":"awhollynewcollection_0_shard5_replica_n38",
"base_url":"http://127.0.0.1:35026/solr",
"node_name":"127.0.0.1:35026_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"6",
"autoAddReplicas":"false",
"nrtReplicas":"4",
"tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34042_solr, 127.0.0.1:35026_solr, 127.0.0.1:43110_solr, 127.0.0.1:44384_solr]
Last available state: DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/5)={
"pullReplicas":"0",
"replicationFactor":"4",
"shards":{
"shard1":{
"range":"80000000-b332ffff",
"state":"active",
"replicas":{
"core_node2":{
"core":"awhollynewcollection_0_shard1_replica_n1",
"base_url":"http://127.0.0.1:34042/solr",
"node_name":"127.0.0.1:34042_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node4":{
"core":"awhollynewcollection_0_shard1_replica_n3",
"base_url":"http://127.0.0.1:44384/solr",
"node_name":"127.0.0.1:44384_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node6":{
"core":"awhollynewcollection_0_shard1_replica_n5",
"base_url":"http://127.0.0.1:43110/solr",
"node_name":"127.0.0.1:43110_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node9":{
"core":"awhollynewcollection_0_shard1_replica_n7",
"base_url":"http://127.0.0.1:35026/solr",
"node_name":"127.0.0.1:35026_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"}}},
"shard2":{
"range":"b3330000-e665ffff",
"state":"active",
"replicas":{
"core_node11":{
"core":"awhollynewcollection_0_shard2_replica_n8",
"base_url":"http://127.0.0.1:34042/solr",
"node_name":"127.0.0.1:34042_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node13":{
"core":"awhollynewcollection_0_shard2_replica_n10",
"base_url":"http://127.0.0.1:44384/solr",
"node_name":"127.0.0.1:44384_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node15":{
"core":"awhollynewcollection_0_shard2_replica_n12",
"base_url":"http://127.0.0.1:43110/solr",
"node_name":"127.0.0.1:43110_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node17":{
"core":"awhollynewcollection_0_shard2_replica_n14",
"base_url":"http://127.0.0.1:35026/solr",
"node_name":"127.0.0.1:35026_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"}}},
"shard3":{
"range":"e6660000-1998ffff",
"state":"active",
"replicas":{
"core_node19":{
"core":"awhollynewcollection_0_shard3_replica_n16",
"base_url":"http://127.0.0.1:34042/solr",
"node_name":"127.0.0.1:34042_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node21":{
"core":"awhollynewcollection_0_shard3_replica_n18",
"base_url":"http://127.0.0.1:44384/solr",
"node_name":"127.0.0.1:44384_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node23":{
"core":"awhollynewcollection_0_shard3_replica_n20",
"base_url":"http://127.0.0.1:43110/solr",
"node_name":"127.0.0.1:43110_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node25":{
"core":"awhollynewcollection_0_shard3_replica_n22",
"base_url":"http://127.0.0.1:35026/solr",
"node_name":"127.0.0.1:35026_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"}}},
"shard4":{
"range":"19990000-4ccbffff",
"state":"active",
"replicas":{
"core_node27":{
"core":"awhollynewcollection_0_shard4_replica_n24",
"base_url":"http://127.0.0.1:34042/solr",
"node_name":"127.0.0.1:34042_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node29":{
"core":"awhollynewcollection_0_shard4_replica_n26",
"base_url":"http://127.0.0.1:44384/solr",
"node_name":"127.0.0.1:44384_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node31":{
"core":"awhollynewcollection_0_shard4_replica_n28",
"base_url":"http://127.0.0.1:43110/solr",
"node_name":"127.0.0.1:43110_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node33":{
"core":"awhollynewcollection_0_shard4_replica_n30",
"base_url":"http://127.0.0.1:35026/solr",
"node_name":"127.0.0.1:35026_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"}}},
"shard5":{
"range":"4ccc0000-7fffffff",
"state":"active",
"replicas":{
"core_node35":{
"core":"awhollynewcollection_0_shard5_replica_n32",
"base_url":"http://127.0.0.1:34042/solr",
"node_name":"127.0.0.1:34042_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node37":{
"core":"awhollynewcollection_0_shard5_replica_n34",
"base_url":"http://127.0.0.1:44384/solr",
"node_name":"127.0.0.1:44384_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node39":{
"core":"awhollynewcollection_0_shard5_replica_n36",
"base_url":"http://127.0.0.1:43110/solr",
"node_name":"127.0.0.1:43110_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"},
"core_node40":{
"core":"awhollynewcollection_0_shard5_replica_n38",
"base_url":"http://127.0.0.1:35026/solr",
"node_name":"127.0.0.1:35026_solr",
"state":"down",
"type":"NRT",
"force_set_state":"false"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"6",
"autoAddReplicas":"false",
"nrtReplicas":"4",
"tlogReplicas":"0"}
at __randomizedtesting.SeedInfo.seed([8836AC67C428561:40F61E727A71AAF4]:0)
at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:758)
at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:764)
at org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:435)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitShardWithRule
Error Message:
Error from server at http://127.0.0.1:33121/gqd/cs: Could not find collection : shardSplitWithRule_rewrite
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:33121/gqd/cs: Could not find collection : shardSplitWithRule_rewrite
at __randomizedtesting.SeedInfo.seed([8836AC67C428561:898DBB74E1B0DE4A]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:649)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1055)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:830)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:763)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:224)
at org.apache.solr.cloud.api.collections.ShardSplitTest.doSplitShardWithRule(ShardSplitTest.java:661)
at org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitShardWithRule(ShardSplitTest.java:628)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.ComputePlanActionTest.testSelectedCollections
Error Message:
Trigger was not fired even after 5 seconds
Stack Trace:
java.lang.AssertionError: Trigger was not fired even after 5 seconds
at __randomizedtesting.SeedInfo.seed([8836AC67C428561:322D8F1F42265C0F]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at org.apache.solr.cloud.autoscaling.ComputePlanActionTest.testSelectedCollections(ComputePlanActionTest.java:501)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.NodeMarkersRegistrationTest.testNodeMarkersRegistration
Error Message:
Path /autoscaling/nodeLost/127.0.0.1:39453_solr exists
Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeLost/127.0.0.1:39453_solr exists
at __randomizedtesting.SeedInfo.seed([8836AC67C428561:1039E2CA7277488E]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at org.junit.Assert.assertFalse(Assert.java:64)
at org.apache.solr.cloud.autoscaling.NodeMarkersRegistrationTest.testNodeMarkersRegistration(NodeMarkersRegistrationTest.java:152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.cdcr.CdcrReplicationHandlerTest.testReplicationWithBufferedUpdates
Error Message:
Timeout while trying to assert number of documents @ source_collection
Stack Trace:
java.lang.AssertionError: Timeout while trying to assert number of documents @ source_collection
at __randomizedtesting.SeedInfo.seed([8836AC67C428561:DB8A3AD839D119F6]:0)
at org.apache.solr.cloud.cdcr.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:277)
at org.apache.solr.cloud.cdcr.CdcrReplicationHandlerTest.testReplicationWithBufferedUpdates(CdcrReplicationHandlerTest.java:233)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.AssertionError: expected:<550> but was:<560>
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.failNotEquals(Assert.java:834)
at org.junit.Assert.assertEquals(Assert.java:645)
at org.junit.Assert.assertEquals(Assert.java:631)
at org.apache.solr.cloud.cdcr.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:268)
... 42 more
FAILED: org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessorTest.testPreemptiveCreation
Error Message:
took over 10 seconds after collection creation to update aliases
Stack Trace:
java.lang.AssertionError: took over 10 seconds after collection creation to update aliases
at __randomizedtesting.SeedInfo.seed([8836AC67C428561:65E9EF6D44206855]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.apache.solr.update.processor.RoutedAliasUpdateProcessorTest.waitColAndAlias(RoutedAliasUpdateProcessorTest.java:77)
at org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessorTest.testPreemptiveCreation(TimeRoutedAliasUpdateProcessorTest.java:326)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13936 lines...]
[junit4] Suite: org.apache.solr.cloud.cdcr.CdcrReplicationHandlerTest
[junit4] 2> 1693375 INFO (SUITE-CdcrReplicationHandlerTest-seed#[8836AC67C428561]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_8836AC67C428561-001/init-core-data-001
[junit4] 2> 1693375 WARN (SUITE-CdcrReplicationHandlerTest-seed#[8836AC67C428561]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=27 numCloses=27
[junit4] 2> 1693375 INFO (SUITE-CdcrReplicationHandlerTest-seed#[8836AC67C428561]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 1693377 INFO (SUITE-CdcrReplicationHandlerTest-seed#[8836AC67C428561]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 1693377 INFO (SUITE-CdcrReplicationHandlerTest-seed#[8836AC67C428561]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /agz/
[junit4] 2> 1693379 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1693379 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1693380 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1693480 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkTestServer start zk server on port:35835
[junit4] 2> 1693480 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:35835
[junit4] 2> 1693480 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 35835
[junit4] 2> 1693491 INFO (zkConnectionManagerCallback-937-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1693497 INFO (zkConnectionManagerCallback-939-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1693510 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-cdcr.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1693511 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 1693512 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1693513 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1693522 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1693523 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1693524 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1693525 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1693526 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1693527 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1693528 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1693529 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testPartialReplicationAfterPeerSync
[junit4] 2> 1693943 WARN (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 1693944 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 1693944 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 1693944 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 1693964 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1693964 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1693964 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 1693966 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2a16a31{/agz,null,AVAILABLE}
[junit4] 2> 1693967 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6382235b{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:41574}
[junit4] 2> 1693967 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.s.Server Started @1694086ms
[junit4] 2> 1693967 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard1, hostContext=/agz, hostPort=41574, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_8836AC67C428561-001/jetty-001/cores}
[junit4] 2> 1693968 ERROR (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1693968 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1693968 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 1693968 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1693968 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1693968 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-03-16T00:53:47.151Z
[junit4] 2> 1694008 INFO (zkConnectionManagerCallback-941-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1694009 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1694009 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_8836AC67C428561-001/jetty-001/solr.xml
[junit4] 2> 1694029 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1694029 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1694031 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6582be76, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1694147 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 1694148 WARN (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@3e79add2[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1694165 WARN (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@24a8ad6a[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1694166 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35835/solr
[junit4] 2> 1694174 INFO (zkConnectionManagerCallback-948-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1694199 INFO (zkConnectionManagerCallback-950-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1694363 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [n:127.0.0.1:41574_agz ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:41574_agz
[junit4] 2> 1694364 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [n:127.0.0.1:41574_agz ] o.a.s.c.Overseer Overseer (id=74353649600233476-127.0.0.1:41574_agz-n_0000000000) starting
[junit4] 2> 1694378 INFO (zkConnectionManagerCallback-957-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1694381 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [n:127.0.0.1:41574_agz ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35835/solr ready
[junit4] 2> 1694382 INFO (OverseerStateUpdate-74353649600233476-127.0.0.1:41574_agz-n_0000000000) [n:127.0.0.1:41574_agz ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:41574_agz
[junit4] 2> 1694382 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [n:127.0.0.1:41574_agz ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41574_agz
[junit4] 2> 1694384 INFO (zkCallback-956-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1694401 INFO (zkCallback-949-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1694466 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [n:127.0.0.1:41574_agz ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1694502 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [n:127.0.0.1:41574_agz ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6582be76
[junit4] 2> 1694530 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [n:127.0.0.1:41574_agz ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6582be76
[junit4] 2> 1694530 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [n:127.0.0.1:41574_agz ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6582be76
[junit4] 2> 1694539 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [n:127.0.0.1:41574_agz ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_8836AC67C428561-001/jetty-001/cores
[junit4] 2> 1694866 WARN (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 1694866 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 1694866 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 1694866 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 1694896 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1694896 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1694896 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 1694897 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@44741b30{/agz,null,AVAILABLE}
[junit4] 2> 1694897 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5b0f3533{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:46510}
[junit4] 2> 1694897 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.s.Server Started @1695016ms
[junit4] 2> 1694897 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard2, hostContext=/agz, hostPort=46510, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_8836AC67C428561-001/jetty-002/cores}
[junit4] 2> 1694898 ERROR (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1694898 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1694898 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 1694898 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1694898 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1694898 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-03-16T00:53:48.081Z
[junit4] 2> 1694924 INFO (zkConnectionManagerCallback-960-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1694925 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1694926 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_8836AC67C428561-001/jetty-002/solr.xml
[junit4] 2> 1694926 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x1082840182d0006, likely client has closed socket
[junit4] 2> 1694930 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1694930 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1694931 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6582be76, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1695134 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 1695136 WARN (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@7360b744[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1695148 WARN (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@3dfae595[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1695149 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35835/solr
[junit4] 2> 1695151 INFO (zkConnectionManagerCallback-967-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1695241 INFO (zkConnectionManagerCallback-969-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1695253 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1695257 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkController Publish node=127.0.0.1:46510_agz as DOWN
[junit4] 2> 1695257 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1695258 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46510_agz
[junit4] 2> 1695259 INFO (zkCallback-956-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1695259 INFO (zkCallback-968-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1695277 INFO (zkCallback-949-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1695319 INFO (zkConnectionManagerCallback-976-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1695321 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1695330 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35835/solr ready
[junit4] 2> 1695330 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1695380 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6582be76
[junit4] 2> 1695402 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6582be76
[junit4] 2> 1695415 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6582be76
[junit4] 2> 1695416 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_8836AC67C428561-001/jetty-002/cores
[junit4] 2> 1695486 INFO (zkConnectionManagerCallback-982-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1695512 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1695513 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplicationAfterPeerSync-seed#[8836AC67C428561]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35835/solr ready
[junit4] 2> 1695515 INFO (qtp983135161-2370) [n:127.0.0.1:46510_agz ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=tmp_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1695529 INFO (OverseerThreadFactory-910-thread-1-processing-n:127.0.0.1:41574_agz) [n:127.0.0.1:41574_agz ] o.a.s.c.a.c.CreateCollectionCmd Create collection tmp_collection
[junit4] 2> 1695650 WARN (OverseerThreadFactory-910-thread-1-processing-n:127.0.0.1:41574_agz) [n:127.0.0.1:41574_agz ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (tmp_collection) without cores.
[junit4] 2> 1695652 INFO (qtp983135161-2370) [n:127.0.0.1:46510_agz ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 1695674 INFO (qtp983135161-2370) [n:127.0.0.1:46510_agz ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=tmp_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&wt=javabin&version=2} status=0 QTime=159
[junit4] 2> 1695676 INFO (qtp983135161-2371) [n:127.0.0.1:46510_agz ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41574_agz&action=ADDREPLICA&collection=tmp_collection&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1695710 INFO (OverseerCollectionConfigSetProcessor-74353649600233476-127.0.0.1:41574_agz-n_0000000000) [n:127.0.0.1:41574_agz ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1695711 INFO (OverseerThreadFactory-910-thread-2-processing-n:127.0.0.1:41574_agz) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:41574_agz for creating new replica of shard shard1 for collection tmp_collection
[junit4] 2> 1695713 INFO (OverseerThreadFactory-910-thread-2-processing-n:127.0.0.1:41574_agz) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 1695738 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz x:tmp_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=tmp_collection_shard1_replica_n1&action=CREATE&collection=tmp_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1695739 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz x:tmp_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1696774 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 1696799 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.s.IndexSchema [tmp_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 1696993 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1697042 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'tmp_collection_shard1_replica_n1' using configuration from collection tmp_collection, trusted=true
[junit4] 2> 1697043 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.tmp_collection.shard1.replica_n1' (registry 'solr.core.tmp_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6582be76
[junit4] 2> 1697062 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.c.SolrCore [[tmp_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_8836AC67C428561-001/jetty-001/cores/tmp_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_8836AC67C428561-001/jetty-001/cores/tmp_collection_shard1_replica_n1/data/]
[junit4] 2> 1697077 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=5, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3547242953918086]
[junit4] 2> 1697254 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 1697254 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1697256 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1697256 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1697257 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=137916360, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 1697274 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6d197bc1[tmp_collection_shard1_replica_n1] main]
[junit4] 2> 1697275 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1697276 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1697276 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1697281 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.h.CdcrBufferStateManager Created znode /collections/tmp_collection/cdcr/state/buffer
[junit4] 2> 1697285 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.h.CdcrProcessStateManager Created znode /collections/tmp_collection/cdcr/state/process
[junit4] 2> 1697337 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1628121470620139520
[junit4] 2> 1697348 INFO (searcherExecutor-924-thread-1-processing-n:127.0.0.1:41574_agz x:tmp_collection_shard1_replica_n1 c:tmp_collection s:shard1) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.c.SolrCore [tmp_collection_shard1_replica_n1] Registered new searcher Searcher@6d197bc1[tmp_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1697351 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/tmp_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 1697351 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/tmp_collection/leaders/shard1
[junit4] 2> 1697354 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1697354 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1697354 INFO (qtp2129804483-2333) [n:127.0.0.1:41574_agz c:tmp_
[...truncated too long message...]
kspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null1108974144
[copy] Copying 239 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null1108974144
[delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null1108974144
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 79ms :: artifacts dl 12ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 591 minutes 24 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2847)
at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2726)
at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2707)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3086)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1072)
at hudson.FilePath.act(FilePath.java:1061)
at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1835)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3088)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1074)
at hudson.FilePath.act(FilePath.java:1061)
at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1835)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
[JENKINS] Lucene-Solr-BadApples-NightlyTests-master - Build # 53 -
Still unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-master/53/
7 tests failed.
FAILED: org.apache.solr.prometheus.exporter.SolrExporterIntegrationTest.jvmMetrics
Error Message:
expected:<4> but was:<0>
Stack Trace:
java.lang.AssertionError: expected:<4> but was:<0>
at __randomizedtesting.SeedInfo.seed([C131B4F6089ADB0B:128589191D74179F]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.failNotEquals(Assert.java:834)
at org.junit.Assert.assertEquals(Assert.java:645)
at org.junit.Assert.assertEquals(Assert.java:631)
at org.apache.solr.prometheus.exporter.SolrExporterIntegrationTest.jvmMetrics(SolrExporterIntegrationTest.java:68)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test
Error Message:
Test abandoned because suite timeout was reached.
Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
at __randomizedtesting.SeedInfo.seed([214E6012A7A9C43E]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([214E6012A7A9C43E]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
Error Message:
Captured an uncaught exception in thread: Thread[id=84799, name=Thread-59961, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=84799, name=Thread-59961, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:44757/collection2_shard6_replica_n30
at __randomizedtesting.SeedInfo.seed([214E6012A7A9C43E]:0)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:638)
Caused by: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:44757/collection2_shard6_replica_n30
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.directUpdate(BaseCloudSolrClient.java:499)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:964)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:830)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:763)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:224)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:635)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:44757/collection2_shard6_replica_n30
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:660)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.directUpdate(BaseCloudSolrClient.java:493)
... 6 more
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:548)
... 11 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
Error Message:
Captured an uncaught exception in thread: Thread[id=84800, name=Thread-59962, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=84800, name=Thread-59962, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:39901/collection2_shard4_replica_n17
at __randomizedtesting.SeedInfo.seed([214E6012A7A9C43E]:0)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:638)
Caused by: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:39901/collection2_shard4_replica_n17
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.directUpdate(BaseCloudSolrClient.java:499)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:964)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:830)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:763)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:224)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:635)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:39901/collection2_shard4_replica_n17
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:660)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.directUpdate(BaseCloudSolrClient.java:493)
... 6 more
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:548)
... 11 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
Error Message:
Captured an uncaught exception in thread: Thread[id=84798, name=Thread-59960, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=84798, name=Thread-59960, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:44757/collection2_shard6_replica_n30
at __randomizedtesting.SeedInfo.seed([214E6012A7A9C43E]:0)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:638)
Caused by: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:44757/collection2_shard6_replica_n30
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.directUpdate(BaseCloudSolrClient.java:499)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:964)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:830)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:763)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:224)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:635)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:44757/collection2_shard6_replica_n30
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:660)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.directUpdate(BaseCloudSolrClient.java:493)
... 6 more
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:548)
... 11 more
FAILED: org.apache.solr.search.mlt.SimpleMLTQParserTest.doTest
Error Message:
Exception during query
Stack Trace:
java.lang.RuntimeException: Exception during query
at __randomizedtesting.SeedInfo.seed([214E6012A7A9C43E:860AD8B6CA12D787]:0)
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:933)
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:893)
at org.apache.solr.search.mlt.SimpleMLTQParserTest.doTest(SimpleMLTQParserTest.java:82)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result/doc[1]/str[@name='id'][.='13']
xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">3</int></lst><result name="response" numFound="16" start="0"><doc><str name="id">18</str><arr name="range_facet_l"><long>18</long></arr><int name="id_i1">18</int><arr name="range_facet_l_dv"><long>18</long></arr><arr name="range_facet_i_dv"><int>18</int></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">18</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.708Z</date><long name="_version_">1627533059249143808</long><int name="intDvoDefault">42</int></doc><doc><str name="id">19</str><arr name="range_facet_l"><long>19</long></arr><int name="id_i1">19</int><arr name="range_facet_l_dv"><long>19</long></arr><arr name="range_facet_i_dv"><int>19</int></arr><arr name="lowerfilt"><str>The hose red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The hose red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">19</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.709Z</date><long name="_version_">1627533059250192384</long><int name="intDvoDefault">42</int></doc><doc><str name="id">20</str><arr name="range_facet_l"><long>20</long></arr><int name="id_i1">20</int><arr name="range_facet_l_dv"><long>20</long></arr><arr name="range_facet_i_dv"><int>20</int></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">20</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.714Z</date><long name="_version_">1627533059256483840</long><int name="intDvoDefault">42</int></doc><doc><str name="id">21</str><arr name="range_facet_l"><long>21</long></arr><int name="id_i1">21</int><arr name="range_facet_l_dv"><long>21</long></arr><arr name="range_facet_i_dv"><int>21</int></arr><arr name="lowerfilt"><str>The court red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The court red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">21</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.715Z</date><long name="_version_">1627533059256483841</long><int name="intDvoDefault">42</int></doc><doc><str name="id">16</str><arr name="range_facet_l"><long>16</long></arr><int name="id_i1">16</int><arr name="range_facet_l_dv"><long>16</long></arr><arr name="range_facet_i_dv"><int>16</int></arr><arr name="lowerfilt"><str>The slim red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The slim red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">16</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.700Z</date><long name="_version_">1627533059240755200</long><int name="intDvoDefault">42</int></doc><doc><str name="id">22</str><arr name="range_facet_l"><long>22</long></arr><int name="id_i1">22</int><arr name="range_facet_l_dv"><long>22</long></arr><arr name="range_facet_i_dv"><int>22</int></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">22</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.721Z</date><long name="_version_">1627533059262775296</long><int name="intDvoDefault">42</int></doc><doc><str name="id">23</str><arr name="range_facet_l"><long>23</long></arr><int name="id_i1">23</int><arr name="range_facet_l_dv"><long>23</long></arr><arr name="range_facet_i_dv"><int>23</int></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">23</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.721Z</date><long name="_version_">1627533059262775297</long><int name="intDvoDefault">42</int></doc><doc><str name="id">14</str><arr name="range_facet_l"><long>14</long></arr><int name="id_i1">14</int><arr name="range_facet_l_dv"><long>14</long></arr><arr name="range_facet_i_dv"><int>14</int></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">14</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.692Z</date><long name="_version_">1627533059232366592</long><int name="intDvoDefault">42</int></doc><doc><str name="id">15</str><arr name="range_facet_l"><long>15</long></arr><int name="id_i1">15</int><arr name="range_facet_l_dv"><long>15</long></arr><arr name="range_facet_i_dv"><int>15</int></arr><arr name="lowerfilt"><str>The fat red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The fat red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">15</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.693Z</date><long name="_version_">1627533059233415168</long><int name="intDvoDefault">42</int></doc><doc><str name="id">13</str><arr name="range_facet_l"><long>13</long></arr><int name="id_i1">13</int><arr name="range_facet_l_dv"><long>13</long></arr><arr name="range_facet_i_dv"><int>13</int></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">13</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.682Z</date><long name="_version_">1627533059221880832</long><int name="intDvoDefault">42</int></doc></result>
</response>
request was:q={!mlt+qf%3Dlowerfilt}17
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:926)
... 41 more
Build Log:
[...truncated 15681 lines...]
[junit4] Suite: org.apache.solr.search.mlt.SimpleMLTQParserTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.search.mlt.SimpleMLTQParserTest_214E6012A7A9C43E-001/init-core-data-001
[junit4] 2> 5626470 INFO (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 5626509 INFO (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 5626538 INFO (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.s.IndexSchema [null] Schema name=test
[junit4] 2> 5626688 INFO (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 5626968 INFO (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 5626969 WARN (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@7f36351e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 5626976 WARN (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@6d00d19d[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 5627002 INFO (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 5627002 INFO (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 5627025 INFO (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191b0004
[junit4] 2> 5627037 INFO (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191b0004
[junit4] 2> 5627037 INFO (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191b0004
[junit4] 2> 5627039 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 5627083 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 5627112 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 5627245 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 5627254 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1, trusted=true
[junit4] 2> 5627254 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191b0004
[junit4] 2> 5627254 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.search.mlt.SimpleMLTQParserTest_214E6012A7A9C43E-001/init-core-data-001/]
[junit4] 2> 5627256 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=12, maxMergedSegmentMB=3.125, floorSegmentMB=1.72265625, forceMergeDeletesPctAllowed=21.002133534007367, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0, deletesPctAllowed=39.913155837900234
[junit4] 2> 5627327 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5627327 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5627329 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5627329 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5627330 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@5f0c5956
[junit4] 2> 5627331 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5e4754c8[collection1] main]
[junit4] 2> 5627331 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 5627339 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 5627368 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 5627368 INFO (coreLoadExecutor-19143-thread-1) [ x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1627533059093954560
[junit4] 2> 5627370 INFO (searcherExecutor-19144-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 5627374 INFO (searcherExecutor-19144-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 5627374 INFO (searcherExecutor-19144-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 5627374 INFO (searcherExecutor-19144-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 5627374 INFO (searcherExecutor-19144-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 5627376 INFO (searcherExecutor-19144-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 5627377 INFO (searcherExecutor-19144-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 5627378 INFO (searcherExecutor-19144-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 5627379 INFO (searcherExecutor-19144-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 5627380 INFO (searcherExecutor-19144-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5e4754c8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5627380 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[1 (1627533059105488896)]} 0 1
[junit4] 2> 5627411 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[2 (1627533059110731776)]} 0 27
[junit4] 2> 5627412 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[3 (1627533059140091904)]} 0 0
[junit4] 2> 5627468 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[4 (1627533059140091905)]} 0 56
[junit4] 2> 5627469 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[5 (1627533059199860736)]} 0 0
[junit4] 2> 5627475 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[6 (1627533059200909312)]} 0 5
[junit4] 2> 5627476 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[7 (1627533059206152192)]} 0 0
[junit4] 2> 5627481 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[8 (1627533059207200768)]} 0 5
[junit4] 2> 5627482 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[9 (1627533059213492224)]} 0 0
[junit4] 2> 5627489 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[10 (1627533059214540800)]} 0 6
[junit4] 2> 5627490 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[12 (1627533059220832256)]} 0 0
[junit4] 2> 5627499 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[13 (1627533059221880832)]} 0 9
[junit4] 2> 5627500 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[14 (1627533059232366592)]} 0 0
[junit4] 2> 5627508 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[15 (1627533059233415168)]} 0 7
[junit4] 2> 5627508 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[16 (1627533059240755200)]} 0 0
[junit4] 2> 5627516 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[17 (1627533059241803776)]} 0 7
[junit4] 2> 5627517 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[18 (1627533059249143808)]} 0 0
[junit4] 2> 5627522 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[19 (1627533059250192384)]} 0 5
[junit4] 2> 5627523 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[20 (1627533059256483840)]} 0 0
[junit4] 2> 5627528 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[21 (1627533059256483841)]} 0 5
[junit4] 2> 5627529 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[22 (1627533059262775296)]} 0 0
[junit4] 2> 5627540 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[23 (1627533059262775297)]} 0 10
[junit4] 2> 5627541 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[24 (1627533059275358208)]} 0 0
[junit4] 2> 5627546 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[25 (1627533059275358209)]} 0 4
[junit4] 2> 5627546 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[26 (1627533059280601088)]} 0 0
[junit4] 2> 5627565 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[27 (1627533059281649664)]} 0 18
[junit4] 2> 5627566 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[28 (1627533059301572608)]} 0 0
[junit4] 2> 5627574 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[29 (1627533059301572609)]} 0 7
[junit4] 2> 5627575 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[30 (1627533059309961216)]} 0 0
[junit4] 2> 5627582 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[31 (1627533059311009792)]} 0 7
[junit4] 2> 5627583 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[32 (1627533059319398400)]} 0 0
[junit4] 2> 5627583 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1627533059319398401,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 5627583 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@35711788 commitCommandVersion:1627533059319398401
[junit4] 2> 5627592 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@c67c6f7[collection1] main]
[junit4] 2> 5627593 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 5627595 INFO (searcherExecutor-19144-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@c67c6f7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_g(9.0.0):c24:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=9.0.0, mergeMaxNumSegments=-1, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=merge, mergeFactor=2, os.version=4.4.0-137-generic, timestamp=1552136477766}]:[attributes={Lucene50StoredFieldsFormat.mode=BEST_SPEED}]) Uninverting(_e(9.0.0):c2:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=9.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552136477748}]:[attributes={Lucene50StoredFieldsFormat.mode=BEST_SPEED}]) Uninverting(_f(9.0.0):c2:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=9.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552136477762}]:[attributes={Lucene50StoredFieldsFormat.mode=BEST_SPEED}]) Uninverting(_h(9.0.0):c2:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=9.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552136477771}]:[attributes={Lucene50StoredFieldsFormat.mode=BEST_SPEED}]) Uninverting(_i(9.0.0):c1:[diagnostics={os=Linux, java.vendor=Oracle Corporation, java.version=1.8.0_191, java.vm.version=25.191-b12, lucene.version=9.0.0, os.arch=amd64, java.runtime.version=1.8.0_191-b12, source=flush, os.version=4.4.0-137-generic, timestamp=1552136477778}]:[attributes={Lucene50StoredFieldsFormat.mode=BEST_SPEED}])))}
[junit4] 2> 5627595 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{commit=} 0 11
[junit4] 2> 5627599 INFO (TEST-SimpleMLTQParserTest.doTest-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.S.Request [collection1] webapp=null path=null params={q={!mlt+qf%3Dlowerfilt}17} hits=16 status=0 QTime=3
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=SimpleMLTQParserTest -Dtests.method=doTest -Dtests.seed=214E6012A7A9C43E -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.badapples=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=pt-BR -Dtests.timezone=Asia/Muscat -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.23s J0 | SimpleMLTQParserTest.doTest <<<
[junit4] > Throwable #1: java.lang.RuntimeException: Exception during query
[junit4] > at __randomizedtesting.SeedInfo.seed([214E6012A7A9C43E:860AD8B6CA12D787]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:933)
[junit4] > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:893)
[junit4] > at org.apache.solr.search.mlt.SimpleMLTQParserTest.doTest(SimpleMLTQParserTest.java:82)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result/doc[1]/str[@name='id'][.='13']
[junit4] > xml response was: <?xml version="1.0" encoding="UTF-8"?>
[junit4] > <response>
[junit4] > <lst name="responseHeader"><int name="status">0</int><int name="QTime">3</int></lst><result name="response" numFound="16" start="0"><doc><str name="id">18</str><arr name="range_facet_l"><long>18</long></arr><int name="id_i1">18</int><arr name="range_facet_l_dv"><long>18</long></arr><arr name="range_facet_i_dv"><int>18</int></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">18</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.708Z</date><long name="_version_">1627533059249143808</long><int name="intDvoDefault">42</int></doc><doc><str name="id">19</str><arr name="range_facet_l"><long>19</long></arr><int name="id_i1">19</int><arr name="range_facet_l_dv"><long>19</long></arr><arr name="range_facet_i_dv"><int>19</int></arr><arr name="lowerfilt"><str>The hose red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The hose red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">19</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.709Z</date><long name="_version_">1627533059250192384</long><int name="intDvoDefault">42</int></doc><doc><str name="id">20</str><arr name="range_facet_l"><long>20</long></arr><int name="id_i1">20</int><arr name="range_facet_l_dv"><long>20</long></arr><arr name="range_facet_i_dv"><int>20</int></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">20</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.714Z</date><long name="_version_">1627533059256483840</long><int name="intDvoDefault">42</int></doc><doc><str name="id">21</str><arr name="range_facet_l"><long>21</long></arr><int name="id_i1">21</int><arr name="range_facet_l_dv"><long>21</long></arr><arr name="range_facet_i_dv"><int>21</int></arr><arr name="lowerfilt"><str>The court red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The court red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">21</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.715Z</date><long name="_version_">1627533059256483841</long><int name="intDvoDefault">42</int></doc><doc><str name="id">16</str><arr name="range_facet_l"><long>16</long></arr><int name="id_i1">16</int><arr name="range_facet_l_dv"><long>16</long></arr><arr name="range_facet_i_dv"><int>16</int></arr><arr name="lowerfilt"><str>The slim red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The slim red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">16</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.700Z</date><long name="_version_">1627533059240755200</long><int name="intDvoDefault">42</int></doc><doc><str name="id">22</str><arr name="range_facet_l"><long>22</long></arr><int name="id_i1">22</int><arr name="range_facet_l_dv"><long>22</long></arr><arr name="range_facet_i_dv"><int>22</int></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">22</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.721Z</date><long name="_version_">1627533059262775296</long><int name="intDvoDefault">42</int></doc><doc><str name="id">23</str><arr name="range_facet_l"><long>23</long></arr><int name="id_i1">23</int><arr name="range_facet_l_dv"><long>23</long></arr><arr name="range_facet_i_dv"><int>23</int></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">23</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.721Z</date><long name="_version_">1627533059262775297</long><int name="intDvoDefault">42</int></doc><doc><str name="id">14</str><arr name="range_facet_l"><long>14</long></arr><int name="id_i1">14</int><arr name="range_facet_l_dv"><long>14</long></arr><arr name="range_facet_i_dv"><int>14</int></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">14</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.692Z</date><long name="_version_">1627533059232366592</long><int name="intDvoDefault">42</int></doc><doc><str name="id">15</str><arr name="range_facet_l"><long>15</long></arr><int name="id_i1">15</int><arr name="range_facet_l_dv"><long>15</long></arr><arr name="range_facet_i_dv"><int>15</int></arr><arr name="lowerfilt"><str>The fat red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The fat red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">15</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.693Z</date><long name="_version_">1627533059233415168</long><int name="intDvoDefault">42</int></doc><doc><str name="id">13</str><arr name="range_facet_l"><long>13</long></arr><int name="id_i1">13</int><arr name="range_facet_l_dv"><long>13</long></arr><arr name="range_facet_i_dv"><int>13</int></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><str name="_root_">13</str><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2019-03-09T13:01:17.682Z</date><long name="_version_">1627533059221880832</long><int name="intDvoDefault">42</int></doc></result>
[junit4] > </response>
[junit4] > request was:q={!mlt+qf%3Dlowerfilt}17
[junit4] > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:926)
[junit4] > ... 41 more
[junit4] 2> 5627603 INFO (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=375040241
[junit4] 2> 5627611 INFO (coreCloseExecutor-19149-thread-1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@5f3e4257
[junit4] 2> 5627611 INFO (coreCloseExecutor-19149-thread-1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=5f3e4257
[junit4] 2> 5627611 INFO (coreCloseExecutor-19149-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@22af089: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@6369e7a8
[junit4] 2> 5627628 INFO (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 5627628 INFO (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@37e6641a: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@3d26095c
[junit4] 2> 5627634 INFO (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 5627634 INFO (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7f24f873: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@26f96dad
[junit4] 2> 5627638 INFO (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 5627638 INFO (SUITE-SimpleMLTQParserTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@31376b80: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@a28e360
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.search.mlt.SimpleMLTQParserTest_214E6012A7A9C43E-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene80): {lowerfilt1=PostingsFormat(name=LuceneVarGapFixedInterval), lowerfilt1and2=PostingsFormat(name=LuceneVarGapDocFreqInterval), multiDefault=PostingsFormat(name=LuceneVarGapDocFreqInterval), _root_=PostingsFormat(name=Asserting), lowerfilt=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{range_facet_l_dv=DocValuesFormat(name=Asserting), multiDefault=DocValuesFormat(name=Lucene80), _root_=DocValuesFormat(name=Lucene80), intDefault=DocValuesFormat(name=Lucene80), range_facet_l=DocValuesFormat(name=Lucene80), _version_=DocValuesFormat(name=Lucene80), lowerfilt1and2=DocValuesFormat(name=Lucene80), id_i1=DocValuesFormat(name=Lucene80), lowerfilt=DocValuesFormat(name=Lucene80), range_facet_i_dv=DocValuesFormat(name=Lucene80), id=DocValuesFormat(name=Asserting), intDvoDefault=DocValuesFormat(name=Asserting), timestamp=DocValuesFormat(name=Lucene80)}, maxPointsInLeafNode=1909, maxMBSortInHeap=5.060265910569069, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@1a95d4e7), locale=pt-BR, timezone=Asia/Muscat
[junit4] 2> NOTE: Linux 4.4.0-137-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=185037776,total=517472256
[junit4] 2> NOTE: All tests run in this JVM: [CdcrBidirectionalTest, TestCSVLoader, TestSearchPerf, AsyncCallRequestStatusResponseTest, DistributedQueueTest, DistributedQueryElevationComponentTest, TestHdfsBackupRestoreCore, RequiredFieldsTest, StressHdfsTest, ConnectionManagerTest, HdfsDirectoryTest, SpellingQueryConverterTest, ConcurrentDeleteAndCreateCollectionTest, DistributedFacetExistsSmallTest, TestCloudPivotFacet, TestNestedUpdateProcessor, BooleanFieldTest, TestTrackingShardHandlerFactory, TestPolicyCloud, TestXmlQParserPlugin, SuggesterFSTTest, TestInitQParser, TestSimpleTextCodec, FullHLLTest, TestNumericTokenStream, TestLegacyNumericRangeQueryBuilder, TestPHPSerializedResponseWriter, TestTlogReplica, SuggesterWFSTTest, CursorPagingTest, CleanupOldIndexTest, SolrMetricReporterTest, CoreMergeIndexesAdminHandlerTest, TestCoreContainer, TlogReplayBufferedWhileIndexingTest, TestLMDirichletSimilarityFactory, ExitableDirectoryReaderTest, ClassificationUpdateProcessorFactoryTest, TestRandomDVFaceting, RecoveryAfterSoftCommitTest, HttpPartitionWithTlogReplicasTest, TestSolrQueryParser, OutOfBoxZkACLAndCredentialsProvidersTest, TestOverriddenPrefixQueryForCustomFieldType, DebugComponentTest, TestExpandComponent, BigEndianAscendingWordSerializerTest, ShufflingReplicaListTransformerTest, SolrPluginUtilsTest, TestHalfAndHalfDocValues, TestNumericTerms64, TestSolrCloudWithHadoopAuthPlugin, TestDFISimilarityFactory, TestCustomStream, TestAuthorizationFramework, TestUniqueKeyFieldResource, ReplaceNodeNoTargetTest, SolrRrdBackendFactoryTest, CircularListTest, TestLRUStatsCache, HdfsTlogReplayBufferedWhileIndexingTest, TestNonDefinedSimilarityFactory, TestExceedMaxTermLength, CloneFieldUpdateProcessorFactoryTest, IndexSchemaTest, TestDistributedMap, CoreAdminHandlerTest, TestSchemaManager, TestDeprecatedFilters, RegexBoostProcessorTest, V2StandaloneTest, ResponseBuilderTest, ReturnFieldsTest, MoveReplicaHDFSFailoverTest, TestSolrConfigHandlerConcurrent, TriggerIntegrationTest, PreAnalyzedFieldManagedSchemaCloudTest, SuggestComponentContextFilterQueryTest, ScheduledMaintenanceTriggerTest, AddSchemaFieldsUpdateProcessorFactoryTest, NodeAddedTriggerIntegrationTest, TestSolrCloudWithDelegationTokens, TestObjectReleaseTracker, TestLegacyFieldReuse, TestBlendedInfixSuggestions, DirectSolrConnectionTest, TestStressLucene, TestCodecSupport, TestDocTermOrdsUninvertLimit, SolrCloudExampleTest, TestConfigSetImmutable, DistanceFunctionTest, TestDynamicURP, AddReplicaTest, SolrSlf4jReporterTest, CollectionTooManyReplicasTest, HighlighterTest, TestPrepRecovery, DirectUpdateHandlerTest, DeleteInactiveReplicaTest, LeaderElectionTest, EmbeddedSolrNoSerializeTest, TestGroupingSearch, TestManagedSchemaThreadSafety, TestQuerySenderNoQuery, TestJsonFacetsWithNestedObjects, TestSimExecutePlanAction, TestMergePolicyConfig, TestSolrCoreSnapshots, TestBinaryResponseWriter, TestGeoJSONResponseWriter, TestSolrDeletionPolicy2, TestJsonFacetRefinement, IndexSizeTriggerTest, TestUtils, ZkStateWriterTest, LukeRequestHandlerTest, SystemInfoHandlerTest, TestSmileRequest, BlockJoinFacetRandomTest, DistributedTermsComponentTest, ForceLeaderTest, TestManagedSynonymGraphFilterFactory, TestRandomFlRTGCloud, TestUnInvertedFieldException, TestInitParams, TestCustomSort, TestSubQueryTransformerCrossCore, TestHashPartitioner, TestComplexPhraseLeadingWildcard, AnalysisAfterCoreReloadTest, MetricsHistoryHandlerTest, TestFuzzyAnalyzedSuggestions, MetricsHistoryIntegrationTest, OrderedExecutorTest, JWTAuthPluginIntegrationTest, TestMultiValuedNumericRangeQuery, TestIntervalFaceting, TestDeleteCollectionOnDownNodes, TestLockTree, TestMiniSolrCloudClusterSSL, TestSegmentSorting, TestSolrCloudWithSecureImpersonation, TestStressInPlaceUpdates, TestStressLiveNodes, UnloadDistributedZkTest, ZkControllerTest, ZkFailoverTest, ZkSolrClientTest, HdfsCollectionsAPIDistributedZkTest, SimpleCollectionCreateDeleteTest, TestCollectionsAPIViaSolrCloudCluster, TestHdfsCloudBackupRestore, TestReplicaProperties, TestRequestStatusCollectionAPI, AutoAddReplicasIntegrationTest, AutoAddReplicasPlanActionTest, AutoScalingHandlerTest, ExecutePlanActionTest, HdfsAutoAddReplicasIntegrationTest, HttpTriggerListenerTest, NodeLostTriggerIntegrationTest, NodeLostTriggerTest, NodeMarkersRegistrationTest, RestoreTriggerStateTest, SearchRateTriggerIntegrationTest, SearchRateTriggerTest, SystemLogListenerTest, TriggerEventQueueTest, TestSimClusterStateProvider, TestSimComputePlanAction, TestSimDistribStateManager, TestSimDistributedQueue, TestSimGenericDistributedQueue, TestSimNodeAddedTrigger, TestSimNodeLostTrigger, TestSimPolicyCloud, TestSimTriggerIntegration, BaseCdcrDistributedZkTest, CdcrOpsAndBoundariesTest, HDFSCollectionsAPITest, HdfsBasicDistributedZk2Test, HdfsBasicDistributedZkTest, TestLazyCores, TestSolrXml, TestXIncludeConfig, CSVRequestHandlerTest, DocumentAnalysisRequestHandlerTest, FieldAnalysisRequestHandlerTest, MoreLikeThisHandlerTest, PingRequestHandlerTest, TestReplicationHandlerDiskOverFlow, TestReqParamsAPI, TestSQLHandlerNonCloud, TestSolrConfigHandlerCloud, V2ApiIntegrationTest, AdminHandlersProxyTest, CoreAdminCreateDiscoverTest, LoggingHandlerTest, MetricsHandlerTest, SecurityConfHandlerTest, SegmentsInfoRequestHandlerTest, ShowFileRequestHandlerTest, StatsReloadRaceTest, TestConfigsApi, TestCoreAdminApis, ZookeeperStatusHandlerTest, BadComponentTest, CustomHighlightComponentTest, CustomTermsComponentTest, DistributedFacetPivotLongTailTest, DistributedFacetPivotWhiteBoxTest, DistributedMLTComponentTest, PhrasesIdentificationComponentTest, ResponseLogComponentTest, SearchHandlerTest, TermVectorComponentTest, TermsComponentTest, TestPivotHelperCode, TaggerTest, TaggingAttributeTest, FastVectorHighlighterTest, TestUnifiedSolrHighlighter, CheckHdfsIndexTest, SmileWriterTest, TestRetrieveFieldsOptimizer, TestChildDocTransformer, TestSubQueryTransformer, TestRestManager, TestBulkSchemaAPI, TestSchemaVersionResource, CopyFieldTest, CurrencyFieldTypeTest, DocValuesMultiTest, DocValuesTest, TestBinaryField, TestBulkSchemaConcurrent, TestCollationField, TestManagedSchemaAPI, TestPointFields, AnalyticsQueryTest, LargeFieldTest, MergeStrategyTest, SortSpecParsingTest, TestAddFieldRealTimeGet, TestCollapseQParserPlugin, TestFastLRUCache, TestFiltering, TestMultiWordSynonyms, TestQueryTypes, TestRTGBase, TestRangeQuery, TestReRankQParserPlugin, TestRealTimeGet, TestRecoveryHdfs, TestSimpleQParserPlugin, TestSolr4Spatial2, TestSolrCoreParser, TestSolrFieldCacheBean, TestStressUserVersions, TestStressVersions, TestTrieFacet, TestValueSourceCache, SpatialHeatmapFacetsTest, TestOrdValues, GraphQueryTest, TestNestedDocsSort, TestScoreJoinQPScore, TestJsonRequest, SimpleMLTQParserTest]
[junit4] Completed [759/845 (1!)] on J0 in 1.26s, 1 test, 1 error <<< FAILURES!
[...truncated 325 lines...]
[junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_214E6012A7A9C43E-001/init-core-data-001
[junit4] 2> 2292848 WARN (SUITE-FullSolrCloudDistribCmdsTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=50 numCloses=50
[junit4] 2> 2292848 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 2292851 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 2292851 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 2292851 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[214E6012A7A9C43E]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 2292854 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2292855 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2292855 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2292955 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.ZkTestServer start zk server on port:33645
[junit4] 2> 2292955 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:33645
[junit4] 2> 2292955 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 33645
[junit4] 2> 2292959 INFO (zkConnectionManagerCallback-9210-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2292961 INFO (zkConnectionManagerCallback-9212-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2292963 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 2292964 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 2292965 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 2292966 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 2292968 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 2292969 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 2292970 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 2292971 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 2292972 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 2292974 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 2292975 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 2292976 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 2293157 WARN (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 2293157 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 2293157 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 2293157 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 2293159 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2293159 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2293159 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 2293159 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1a244f5d{/,null,AVAILABLE}
[junit4] 2> 2293160 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@63fa5362{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:36564}
[junit4] 2> 2293160 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.e.j.s.Server Started @2293222ms
[junit4] 2> 2293160 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_214E6012A7A9C43E-001/tempDir-001/control/data, hostContext=/, hostPort=36564, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_214E6012A7A9C43E-001/control-001/cores}
[junit4] 2> 2293161 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2293161 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 2293161 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 2293161 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2293161 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2293161 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-03-09T12:05:43.354Z
[junit4] 2> 2293167 INFO (zkConnectionManagerCallback-9214-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2293168 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2293168 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_214E6012A7A9C43E-001/control-001/solr.xml
[junit4] 2> 2293173 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2293173 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2293175 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@36b802d, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2293314 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 2293316 WARN (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@3b3dc58e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2293320 WARN (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@3aabd9a0[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2293321 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33645/solr
[junit4] 2> 2293323 INFO (zkConnectionManagerCallback-9220-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2293325 INFO (zkConnectionManagerCallback-9222-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2293412 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [n:127.0.0.1:36564_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:36564_
[junit4] 2> 2293413 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [n:127.0.0.1:36564_ ] o.a.s.c.Overseer Overseer (id=72927119166865412-127.0.0.1:36564_-n_0000000000) starting
[junit4] 2> 2293419 INFO (zkConnectionManagerCallback-9229-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2293422 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [n:127.0.0.1:36564_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33645/solr ready
[junit4] 2> 2293423 INFO (OverseerStateUpdate-72927119166865412-127.0.0.1:36564_-n_0000000000) [n:127.0.0.1:36564_ ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:36564_
[junit4] 2> 2293423 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [n:127.0.0.1:36564_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36564_
[junit4] 2> 2293425 INFO (OverseerStateUpdate-72927119166865412-127.0.0.1:36564_-n_0000000000) [n:127.0.0.1:36564_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2293433 INFO (zkCallback-9228-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2293465 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [n:127.0.0.1:36564_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 2293492 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [n:127.0.0.1:36564_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36b802d
[junit4] 2> 2293507 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [n:127.0.0.1:36564_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36b802d
[junit4] 2> 2293507 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [n:127.0.0.1:36564_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36b802d
[junit4] 2> 2293509 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [n:127.0.0.1:36564_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_214E6012A7A9C43E-001/control-001/cores
[junit4] 2> 2293538 INFO (zkConnectionManagerCallback-9235-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2293540 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2293541 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[214E6012A7A9C43E]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33645/solr ready
[junit4] 2> 2293543 INFO (qtp248155231-84194) [n:127.0.0.1:36564_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:36564_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2293546 INFO (OverseerThreadFactory-9144-thread-1-processing-n:127.0.0.1:36564_) [n:127.0.0.1:36564_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 2293654 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2293654 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2294676 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 2294692 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 2294798 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 2294815 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 2294815 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36b802d
[junit4] 2> 2294815 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_214E6012A7A9C43E-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_214E6012A7A9C43E-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 2294818 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=33, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6640916283049024]
[junit4] 2> 2294823 WARN (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2294885 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2294885 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2294887 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2294887 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2294889 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=34.0234375, floorSegmentMB=1.6044921875, forceMergeDeletesPctAllowed=7.530456859601676, segmentsPerTier=27.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0, deletesPctAllowed=28.524934377383595
[junit4] 2> 2294889 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4cde54e9[control_collection_shard1_replica_n1] main]
[junit4] 2> 2294890 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2294891 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2294891 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2294891 INFO (qtp248155231-84196) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1627529564739600384
[junit4] 2> 2294895 INFO (searcherExecutor-9149-thread-1-processing-n:127.0.0.1:36564_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:36564_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@4cde54e9[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectory
[...truncated too long message...]
file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null195981466
[copy] Copying 239 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null195981466
[delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null195981466
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 50ms :: artifacts dl 6ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 387 minutes 4 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2847)
at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2726)
at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2707)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3086)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1072)
at hudson.FilePath.act(FilePath.java:1061)
at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1835)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3088)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1074)
at hudson.FilePath.act(FilePath.java:1061)
at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1835)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
[JENKINS] Lucene-Solr-BadApples-NightlyTests-master - Build # 52 -
Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-master/52/
No tests ran.
Build Log:
[...truncated 10 lines...]
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from https://gitbox.apache.org/repos/asf/lucene-solr.git
at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:894)
at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1161)
at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1192)
at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
at hudson.scm.SCM.checkout(SCM.java:504)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1810)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git fetch --tags --progress https://gitbox.apache.org/repos/asf/lucene-solr.git +refs/heads/*:refs/remotes/origin/*" returned status code 128:
stdout:
stderr: fatal: unable to access 'https://gitbox.apache.org/repos/asf/lucene-solr.git/': gnutls_handshake() failed: A TLS packet with unexpected length was received.
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2042)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandWithCredentials(CliGitAPIImpl.java:1761)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.access$400(CliGitAPIImpl.java:72)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl$1.execute(CliGitAPIImpl.java:442)
at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler$1.call(RemoteGitImpl.java:153)
at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler$1.call(RemoteGitImpl.java:146)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler.execute(RemoteGitImpl.java:146)
at sun.reflect.GeneratedMethodAccessor1199.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler.invoke(RemoteGitImpl.java:132)
at com.sun.proxy.$Proxy116.execute(Unknown Source)
at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:892)
at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1161)
at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1192)
at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
at hudson.scm.SCM.checkout(SCM.java:504)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1810)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
No credentials specified
> git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
> git config remote.origin.url https://gitbox.apache.org/repos/asf/lucene-solr.git # timeout=10
Fetching upstream changes from https://gitbox.apache.org/repos/asf/lucene-solr.git
> git --version # timeout=10
> git fetch --tags --progress https://gitbox.apache.org/repos/asf/lucene-solr.git +refs/heads/*:refs/remotes/origin/*
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from https://gitbox.apache.org/repos/asf/lucene-solr.git
at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:894)
at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1161)
at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1192)
at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
at hudson.scm.SCM.checkout(SCM.java:504)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1810)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git fetch --tags --progress https://gitbox.apache.org/repos/asf/lucene-solr.git +refs/heads/*:refs/remotes/origin/*" returned status code 128:
stdout:
stderr: fatal: unable to access 'https://gitbox.apache.org/repos/asf/lucene-solr.git/': gnutls_handshake() failed: A TLS packet with unexpected length was received.
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2042)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandWithCredentials(CliGitAPIImpl.java:1761)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.access$400(CliGitAPIImpl.java:72)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl$1.execute(CliGitAPIImpl.java:442)
at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler$1.call(RemoteGitImpl.java:153)
at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler$1.call(RemoteGitImpl.java:146)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler.execute(RemoteGitImpl.java:146)
at sun.reflect.GeneratedMethodAccessor1199.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler.invoke(RemoteGitImpl.java:132)
at com.sun.proxy.$Proxy116.execute(Unknown Source)
at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:892)
at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1161)
at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1192)
at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
at hudson.scm.SCM.checkout(SCM.java:504)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1810)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Retrying after 10 seconds
No credentials specified
> git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
> git config remote.origin.url https://gitbox.apache.org/repos/asf/lucene-solr.git # timeout=10
Fetching upstream changes from https://gitbox.apache.org/repos/asf/lucene-solr.git
> git --version # timeout=10
> git fetch --tags --progress https://gitbox.apache.org/repos/asf/lucene-solr.git +refs/heads/*:refs/remotes/origin/*
ERROR: Error fetching remote repo 'origin'
hudson.plugins.git.GitException: Failed to fetch from https://gitbox.apache.org/repos/asf/lucene-solr.git
at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:894)
at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1161)
at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1192)
at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
at hudson.scm.SCM.checkout(SCM.java:504)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1810)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused by: hudson.plugins.git.GitException: Command "git fetch --tags --progress https://gitbox.apache.org/repos/asf/lucene-solr.git +refs/heads/*:refs/remotes/origin/*" returned status code 128:
stdout:
stderr: fatal: unable to access 'https://gitbox.apache.org/repos/asf/lucene-solr.git/': gnutls_handshake() failed: A TLS packet with unexpected length was received.
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandIn(CliGitAPIImpl.java:2042)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.launchCommandWithCredentials(CliGitAPIImpl.java:1761)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl.access$400(CliGitAPIImpl.java:72)
at org.jenkinsci.plugins.gitclient.CliGitAPIImpl$1.execute(CliGitAPIImpl.java:442)
at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler$1.call(RemoteGitImpl.java:153)
at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler$1.call(RemoteGitImpl.java:146)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Suppressed: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler.execute(RemoteGitImpl.java:146)
at sun.reflect.GeneratedMethodAccessor1199.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.jenkinsci.plugins.gitclient.RemoteGitImpl$CommandInvocationHandler.invoke(RemoteGitImpl.java:132)
at com.sun.proxy.$Proxy116.execute(Unknown Source)
at hudson.plugins.git.GitSCM.fetchFrom(GitSCM.java:892)
at hudson.plugins.git.GitSCM.retrieveChanges(GitSCM.java:1161)
at hudson.plugins.git.GitSCM.checkout(GitSCM.java:1192)
at org.jenkinsci.plugins.multiplescms.MultiSCM.checkout(MultiSCM.java:143)
at hudson.scm.SCM.checkout(SCM.java:504)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1208)
at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:574)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:86)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:499)
at hudson.model.Run.execute(Run.java:1810)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
ERROR: Error fetching remote repo 'origin'
Archiving artifacts
Recording test results
ERROR: Step ‘Publish JUnit test result report’ failed: Test reports were found but none of them are new. Did leafNodes run?
For example, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/build/analysis/common/test/TEST-org.apache.lucene.analysis.ar.TestArabicAnalyzer.xml is 6 days 11 hr old
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-BadApples-NightlyTests-master - Build # 51 -
Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-master/51/
2 tests failed.
FAILED: org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest.test
Error Message:
document count mismatch. control=1638 sum(shards)=1652 cloudClient=1652
Stack Trace:
java.lang.AssertionError: document count mismatch. control=1638 sum(shards)=1652 cloudClient=1652
at __randomizedtesting.SeedInfo.seed([29E266F0FBEBE94:8ACA19B5A142D36C]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1499)
at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:266)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.hdfs.StressHdfsTest.test
Error Message:
Error from server at http://127.0.0.1:34861/_/l/delete_data_dir: Async exception during distributed update: Error from server at http://127.0.0.1:39225/_/l/delete_data_dir_shard2_replica_n2/: null request: http://127.0.0.1:39225/_/l/delete_data_dir_shard2_replica_n2/ Remote error message: Server error writing document id 1404 to the index
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:34861/_/l/delete_data_dir: Async exception during distributed update: Error from server at http://127.0.0.1:39225/_/l/delete_data_dir_shard2_replica_n2/: null
request: http://127.0.0.1:39225/_/l/delete_data_dir_shard2_replica_n2/
Remote error message: Server error writing document id 1404 to the index
at __randomizedtesting.SeedInfo.seed([29E266F0FBEBE94:8ACA19B5A142D36C]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:650)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:256)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:245)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:177)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:156)
at org.apache.solr.cloud.hdfs.StressHdfsTest.createAndDeleteCollection(StressHdfsTest.java:193)
at org.apache.solr.cloud.hdfs.StressHdfsTest.test(StressHdfsTest.java:108)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13745 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/init-core-data-001
[junit4] 2> 1219087 WARN (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=100 numCloses=100
[junit4] 2> 1219087 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 1219095 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 1219095 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 1219096 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /mz/x
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 1219226 WARN (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 1219295 WARN (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1219298 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 1219351 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1219351 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1219351 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 1219371 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@547acdc5{static,/static,jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
[junit4] 2> 1219590 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@7e8477d5{hdfs,/,file:///x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/jetty-localhost-38583-hdfs-_-any-5748904265369993100.dir/webapp/,AVAILABLE}{/hdfs}
[junit4] 2> 1219591 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.AbstractConnector Started ServerConnector@51067aeb{HTTP/1.1,[http/1.1]}{localhost:38583}
[junit4] 2> 1219591 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.Server Started @1219644ms
[junit4] 2> 1219733 WARN (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.a.h.h.s.c.MetricsLoggerTask Metrics logging will not be async since the logger is not log4j
[junit4] 2> 1219798 WARN (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1219802 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 1219804 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1219804 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1219804 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1219805 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2b8e698{static,/static,jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
[junit4] 2> 1220027 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@3ec8acba{datanode,/,file:///x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/jetty-localhost-39173-datanode-_-any-4092390915426280258.dir/webapp/,AVAILABLE}{/datanode}
[junit4] 2> 1220027 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7c478c1d{HTTP/1.1,[http/1.1]}{localhost:39173}
[junit4] 2> 1220028 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.Server Started @1220081ms
[junit4] 2> 1220046 WARN (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.a.h.h.s.c.MetricsLoggerTask Metrics logging will not be async since the logger is not log4j
[junit4] 2> 1220098 WARN (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1220099 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 1220100 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1220100 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1220100 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1220101 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4956b200{static,/static,jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
[junit4] 2> 1220268 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x7b6fa862ba7d8158: Processing first storage report for DS-a82881b2-32d2-45b2-b546-85b122c5cc1b from datanode b468c6ca-86f7-474e-8a52-590b31ac10c0
[junit4] 2> 1220268 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x7b6fa862ba7d8158: from storage DS-a82881b2-32d2-45b2-b546-85b122c5cc1b node DatanodeRegistration(127.0.0.1:39481, datanodeUuid=b468c6ca-86f7-474e-8a52-590b31ac10c0, infoPort=37426, infoSecurePort=0, ipcPort=41367, storageInfo=lv=-57;cid=testClusterID;nsid=60639002;c=1550937508556), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
[junit4] 2> 1220268 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x7b6fa862ba7d8158: Processing first storage report for DS-b40f295b-4ce9-4bcc-b301-6edec372057c from datanode b468c6ca-86f7-474e-8a52-590b31ac10c0
[junit4] 2> 1220268 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x7b6fa862ba7d8158: from storage DS-b40f295b-4ce9-4bcc-b301-6edec372057c node DatanodeRegistration(127.0.0.1:39481, datanodeUuid=b468c6ca-86f7-474e-8a52-590b31ac10c0, infoPort=37426, infoSecurePort=0, ipcPort=41367, storageInfo=lv=-57;cid=testClusterID;nsid=60639002;c=1550937508556), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
[junit4] 2> 1220322 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@5ce6f113{datanode,/,file:///x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/jetty-localhost-34591-datanode-_-any-6381145968145431572.dir/webapp/,AVAILABLE}{/datanode}
[junit4] 2> 1220322 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.AbstractConnector Started ServerConnector@68c84b79{HTTP/1.1,[http/1.1]}{localhost:34591}
[junit4] 2> 1220322 INFO (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.Server Started @1220375ms
[junit4] 2> 1220344 WARN (SUITE-HdfsChaosMonkeyNothingIsSafeTest-seed#[29E266F0FBEBE94]-worker) [ ] o.a.h.h.s.c.MetricsLoggerTask Metrics logging will not be async since the logger is not log4j
[junit4] 2> 1220524 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x747d87db2230e0ff: Processing first storage report for DS-cca828ee-3fbe-4d59-aaa0-8192a8587ccb from datanode 96a74d7d-060a-4eae-bcad-9b4d82475fb1
[junit4] 2> 1220524 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x747d87db2230e0ff: from storage DS-cca828ee-3fbe-4d59-aaa0-8192a8587ccb node DatanodeRegistration(127.0.0.1:40794, datanodeUuid=96a74d7d-060a-4eae-bcad-9b4d82475fb1, infoPort=40756, infoSecurePort=0, ipcPort=41232, storageInfo=lv=-57;cid=testClusterID;nsid=60639002;c=1550937508556), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
[junit4] 2> 1220524 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x747d87db2230e0ff: Processing first storage report for DS-475abb34-a407-4879-8098-5066cb3237e3 from datanode 96a74d7d-060a-4eae-bcad-9b4d82475fb1
[junit4] 2> 1220524 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x747d87db2230e0ff: from storage DS-475abb34-a407-4879-8098-5066cb3237e3 node DatanodeRegistration(127.0.0.1:40794, datanodeUuid=96a74d7d-060a-4eae-bcad-9b4d82475fb1, infoPort=40756, infoSecurePort=0, ipcPort=41232, storageInfo=lv=-57;cid=testClusterID;nsid=60639002;c=1550937508556), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
[junit4] 2> 1220687 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1220692 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1220692 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1220792 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkTestServer start zk server on port:44434
[junit4] 2> 1220792 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:44434
[junit4] 2> 1220792 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 44434
[junit4] 2> 1220825 INFO (zkConnectionManagerCallback-7803-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1220838 INFO (zkConnectionManagerCallback-7805-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1220839 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1220841 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 1220842 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1220843 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1220844 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1220845 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1220846 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1220848 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1220849 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1220850 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1220851 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1220852 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 1221065 WARN (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 1221065 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 1221065 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 1221065 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 1221066 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1221066 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1221066 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1221069 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@116a7fb8{/mz/x,null,AVAILABLE}
[junit4] 2> 1221069 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2bb26687{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:38515}
[junit4] 2> 1221069 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.e.j.s.Server Started @1221122ms
[junit4] 2> 1221069 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:38559/hdfs__localhost_38559__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-BadApples-NightlyTests-master_checkout_solr_build_solr-core_test_J2_temp_solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001_tempDir-002_control_data, hostContext=/mz/x, hostPort=38515, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/control-001/cores}
[junit4] 2> 1221070 ERROR (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1221070 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1221070 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 1221070 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1221070 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1221070 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-02-23T15:58:30.460Z
[junit4] 2> 1221072 INFO (zkConnectionManagerCallback-7807-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1221073 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1221073 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/control-001/solr.xml
[junit4] 2> 1221078 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1221078 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1221080 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1221164 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 1221166 WARN (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@25acbd01[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1221171 WARN (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@69f9ab58[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1221172 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44434/solr
[junit4] 2> 1221174 INFO (zkConnectionManagerCallback-7813-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1221176 INFO (zkConnectionManagerCallback-7815-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1221304 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38515_mz%2Fx
[junit4] 2> 1221305 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.c.Overseer Overseer (id=72848762358202372-127.0.0.1:38515_mz%2Fx-n_0000000000) starting
[junit4] 2> 1221319 INFO (zkConnectionManagerCallback-7822-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1221323 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44434/solr ready
[junit4] 2> 1221324 INFO (OverseerStateUpdate-72848762358202372-127.0.0.1:38515_mz%2Fx-n_0000000000) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:38515_mz%2Fx
[junit4] 2> 1221325 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38515_mz%2Fx
[junit4] 2> 1221333 INFO (zkCallback-7814-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1221345 INFO (zkCallback-7821-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1221350 DEBUG (OverseerAutoScalingTriggerThread-72848762358202372-127.0.0.1:38515_mz%2Fx-n_0000000000) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:38515_mz%2Fx]
[junit4] 2> 1221352 DEBUG (ScheduledTrigger-2980-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 1221354 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1221381 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e
[junit4] 2> 1221395 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e
[junit4] 2> 1221396 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e
[junit4] 2> 1221397 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/control-001/cores
[junit4] 2> 1221434 INFO (zkConnectionManagerCallback-7828-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1221435 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1221437 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44434/solr ready
[junit4] 2> 1221438 INFO (qtp787767952-16090) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38515_mz%252Fx&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1221443 INFO (OverseerThreadFactory-2982-thread-1-processing-n:127.0.0.1:38515_mz%2Fx) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 1221567 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1221568 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1222354 DEBUG (ScheduledTrigger-2980-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 1222596 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 1222612 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 1222724 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1222748 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 1222748 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e
[junit4] 2> 1222748 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:38559/solr_hdfs_home
[junit4] 2> 1222748 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 1222748 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://localhost:38559/solr_hdfs_home/control_collection/core_node2/data/]
[junit4] 2> 1222750 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:38559/solr_hdfs_home/control_collection/core_node2/data/snapshot_metadata
[junit4] 2> 1222760 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 1222760 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 1222760 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new global HDFS BlockCache
[junit4] 2> 1222778 WARN (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 1223355 DEBUG (ScheduledTrigger-2980-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 1224355 DEBUG (ScheduledTrigger-2980-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 1225355 DEBUG (ScheduledTrigger-2980-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 1226355 DEBUG (ScheduledTrigger-2980-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 1227355 DEBUG (ScheduledTrigger-2980-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 1227779 WARN (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 1228355 DEBUG (ScheduledTrigger-2980-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 1229356 DEBUG (ScheduledTrigger-2980-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 1230356 DEBUG (ScheduledTrigger-2980-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 1231356 DEBUG (ScheduledTrigger-2980-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 1232356 DEBUG (ScheduledTrigger-2980-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 1232964 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 1232966 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:38559/solr_hdfs_home/control_collection/core_node2/data
[junit4] 2> 1233019 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:38559/solr_hdfs_home/control_collection/core_node2/data/index
[junit4] 2> 1233036 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 1233036 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 1233040 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 1233041 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@b558f02
[junit4] 2> 1233240 WARN (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1233356 DEBUG (ScheduledTrigger-2980-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 1233411 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 1233411 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1233411 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 1233425 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms;
[junit4] 2> 1233425 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1233436 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=50.7861328125, floorSegmentMB=0.9345703125, forceMergeDeletesPctAllowed=25.986590869294186, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=30.38741289016026
[junit4] 2> 1233476 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@23aaef91[control_collection_shard1_replica_n1] main]
[junit4] 2> 1233478 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1233478 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1233479 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1233504 WARN (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:43337/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:43337 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 1233504 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1626275864006098944
[junit4] 2> 1233516 INFO (searcherExecutor-2987-thread-1-processing-n:127.0.0.1:38515_mz%2Fx x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@23aaef91[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1233521 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 1233521 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
[junit4] 2> 1233537 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1233537 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1233537 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:38515/mz/x/control_collection_shard1_replica_n1/
[junit4] 2> 1233538 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1233538 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:38515/mz/x/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 1233538 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72848762358202372-core_node2-n_0000000000
[junit4] 2> 1233542 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:38515/mz/x/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 1233544 INFO (zkCallback-7814-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1233545 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1233547 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=11979
[junit4] 2> 1233549 INFO (zkCallback-7814-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1233551 INFO (qtp787767952-16090) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 1233650 INFO (zkCallback-7814-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1233650 INFO (zkCallback-7814-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1233651 INFO (qtp787767952-16090) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38515_mz%252Fx&wt=javabin&version=2} status=0 QTime=12213
[junit4] 2> 1233652 INFO (zkCallback-7814-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1233681 INFO (zkConnectionManagerCallback-7834-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1233682 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1233683 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44434/solr ready
[junit4] 2> 1233683 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1233684 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1233695 INFO (OverseerThreadFactory-2982-thread-2-processing-n:127.0.0.1:38515_mz%2Fx) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 1233695 INFO (OverseerCollectionConfigSetProcessor-72848762358202372-127.0.0.1:38515_mz%2Fx-n_0000000000) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1233902 WARN (OverseerThreadFactory-2982-thread-2-processing-n:127.0.0.1:38515_mz%2Fx) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 1233904 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 1233905 INFO (qtp787767952-16092) [n:127.0.0.1:38515_mz%2Fx ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=220
[junit4] 2> 1233906 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=5
[junit4] 2> 1234075 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/shard-1-001 of type NRT
[junit4] 2> 1234090 WARN (closeThreadPool-7835-thread-1) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 1234091 INFO (closeThreadPool-7835-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 1234091 INFO (closeThreadPool-7835-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 1234091 INFO (closeThreadPool-7835-thread-1) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 1234119 INFO (closeThreadPool-7835-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1234119 INFO (closeThreadPool-7835-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1234119 INFO (closeThreadPool-7835-thread-1) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 1234119 INFO (closeThreadPool-7835-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@74f1dd51{/mz/x,null,AVAILABLE}
[junit4] 2> 1234120 INFO (closeThreadPool-7835-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@459354e2{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:41941}
[junit4] 2> 1234120 INFO (closeThreadPool-7835-thread-1) [ ] o.e.j.s.Server Started @1234173ms
[junit4] 2> 1234120 INFO (closeThreadPool-7835-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:38559/hdfs__localhost_38559__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-BadApples-NightlyTests-master_checkout_solr_build_solr-core_test_J2_temp_solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001_tempDir-002_jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/mz/x, hostPort=41941, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/shard-1-001/cores}
[junit4] 2> 1234120 ERROR (closeThreadPool-7835-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1234120 INFO (closeThreadPool-7835-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1234120 INFO (closeThreadPool-7835-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 1234120 INFO (closeThreadPool-7835-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1234120 INFO (closeThreadPool-7835-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1234120 INFO (closeThreadPool-7835-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-02-23T15:58:43.510Z
[junit4] 2> 1234123 INFO (zkConnectionManagerCallback-7837-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1234124 INFO (closeThreadPool-7835-thread-1) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1234124 INFO (closeThreadPool-7835-thread-1) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/shard-1-001/solr.xml
[junit4] 2> 1234128 INFO (closeThreadPool-7835-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1234128 INFO (closeThreadPool-7835-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1234131 INFO (closeThreadPool-7835-thread-1) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1234331 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/shard-2-001 of type NRT
[junit4] 2> 1234351 WARN (closeThreadPool-7835-thread-2) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 1234351 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 1234351 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 1234351 INFO (closeThreadPool-7835-thread-2) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 1234357 DEBUG (ScheduledTrigger-2980-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 1234380 INFO (closeThreadPool-7835-thread-1) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 1234381 INFO (closeThreadPool-7835-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1234381 INFO (closeThreadPool-7835-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1234381 INFO (closeThreadPool-7835-thread-2) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 1234381 INFO (closeThreadPool-7835-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@581c0de8{/mz/x,null,AVAILABLE}
[junit4] 2> 1234382 INFO (closeThreadPool-7835-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4d585c04{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:43852}
[junit4] 2> 1234382 INFO (closeThreadPool-7835-thread-2) [ ] o.e.j.s.Server Started @1234435ms
[junit4] 2> 1234382 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:38559/hdfs__localhost_38559__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-BadApples-NightlyTests-master_checkout_solr_build_solr-core_test_J2_temp_solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001_tempDir-002_jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/mz/x, hostPort=43852, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/shard-2-001/cores}
[junit4] 2> 1234382 ERROR (closeThreadPool-7835-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1234382 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1234383 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 1234383 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1234383 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1234383 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-02-23T15:58:43.773Z
[junit4] 2> 1234385 WARN (closeThreadPool-7835-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@746a1ec[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1234396 INFO (zkConnectionManagerCallback-7840-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1234407 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1234407 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/shard-2-001/solr.xml
[junit4] 2> 1234409 WARN (closeThreadPool-7835-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@795228d1[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1234410 INFO (closeThreadPool-7835-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44434/solr
[junit4] 2> 1234416 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1234416 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1234418 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1234435 INFO (zkConnectionManagerCallback-7845-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1234458 INFO (zkConnectionManagerCallback-7847-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1234464 INFO (closeThreadPool-7835-thread-1) [n:127.0.0.1:41941_mz%2Fx ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1234468 INFO (closeThreadPool-7835-thread-1) [n:127.0.0.1:41941_mz%2Fx ] o.a.s.c.ZkController Publish node=127.0.0.1:41941_mz%2Fx as DOWN
[junit4] 2> 1234470 INFO (closeThreadPool-7835-thread-1) [n:127.0.0.1:41941_mz%2Fx ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1234470 INFO (closeThreadPool-7835-thread-1) [n:127.0.0.1:41941_mz%2Fx ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41941_mz%2Fx
[junit4] 2> 1234471 INFO (zkCallback-7821-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1234472 INFO (zkCallback-7833-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1234472 INFO (zkCallback-7814-thread-3) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1234478 INFO (zkCallback-7846-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1234528 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 1234539 WARN (closeThreadPool-7835-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@8fd0177[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1234542 INFO (zkConnectionManagerCallback-7854-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1234543 INFO (closeThreadPool-7835-thread-1) [n:127.0.0.1:41941_mz%2Fx ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1234544 WARN (closeThreadPool-7835-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@74005405[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1234545 INFO (closeThreadPool-7835-thread-1) [n:127.0.0.1:41941_mz%2Fx ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44434/solr ready
[junit4] 2> 1234545 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44434/solr
[junit4] 2> 1234546 INFO (closeThreadPool-7835-thread-1) [n:127.0.0.1:41941_mz%2Fx ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1234553 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/shard-3-001 of type NRT
[junit4] 2> 1234556 INFO (zkConnectionManagerCallback-7861-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1234563 WARN (closeThreadPool-7835-thread-3) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 1234571 INFO (closeThreadPool-7835-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 1234571 INFO (closeThreadPool-7835-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 1234572 INFO (closeThreadPool-7835-thread-3) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 1234579 INFO (closeThreadPool-7835-thread-1) [n:127.0.0.1:41941_mz%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e
[junit4] 2> 1234595 INFO (closeThreadPool-7835-thread-1) [n:127.0.0.1:41941_mz%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e
[junit4] 2> 1234595 INFO (closeThreadPool-7835-thread-1) [n:127.0.0.1:41941_mz%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e
[junit4] 2> 1234597 INFO (closeThreadPool-7835-thread-1) [n:127.0.0.1:41941_mz%2Fx ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/shard-1-001/cores
[junit4] 2> 1234606 INFO (zkConnectionManagerCallback-7863-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1234611 INFO (closeThreadPool-7835-thread-2) [n:127.0.0.1:43852_mz%2Fx ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1234616 INFO (closeThreadPool-7835-thread-2) [n:127.0.0.1:43852_mz%2Fx ] o.a.s.c.ZkController Publish node=127.0.0.1:43852_mz%2Fx as DOWN
[junit4] 2> 1234624 INFO (closeThreadPool-7835-thread-2) [n:127.0.0.1:43852_mz%2Fx ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1234624 INFO (closeThreadPool-7835-thread-2) [n:127.0.0.1:43852_mz%2Fx ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43852_mz%2Fx
[junit4] 2> 1234626 INFO (zkCallback-7814-thread-3) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1234626 INFO (zkCallback-7821-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1234626 INFO (zkCallback-7833-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1234627 INFO (closeThreadPool-7835-thread-3) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1234627 INFO (closeThreadPool-7835-thread-3) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1234627 INFO (closeThreadPool-7835-thread-3) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 1234627 INFO (closeThreadPool-7835-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3fa4de0c{/mz/x,null,AVAILABLE}
[junit4] 2> 1234633 INFO (zkCallback-7846-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1234634 INFO (zkCallback-7862-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1234634 INFO (closeThreadPool-7835-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7958e2b6{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:44073}
[junit4] 2> 1234634 INFO (closeThreadPool-7835-thread-3) [ ] o.e.j.s.Server Started @1234687ms
[junit4] 2> 1234634 INFO (closeThreadPool-7835-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:38559/hdfs__localhost_38559__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-BadApples-NightlyTests-master_checkout_solr_build_solr-core_test_J2_temp_solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001_tempDir-002_jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/mz/x, hostPort=44073, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/shard-3-001/cores}
[junit4] 2> 1234634 ERROR (closeThreadPool-7835-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1234634 INFO (closeThreadPool-7835-thread-3) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1234634 INFO (closeThreadPool-7835-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 1234635 INFO (closeThreadPool-7835-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1234635 INFO (closeThreadPool-7835-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1234635 INFO (closeThreadPool-7835-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-02-23T15:58:44.025Z
[junit4] 2> 1234635 INFO (zkCallback-7853-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1234637 INFO (zkConnectionManagerCallback-7867-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1234638 INFO (closeThreadPool-7835-thread-3) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1234638 INFO (closeThreadPool-7835-thread-3) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/shard-3-001/solr.xml
[junit4] 2> 1234645 INFO (closeThreadPool-7835-thread-3) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1234645 INFO (closeThreadPool-7835-thread-3) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1234647 INFO (closeThreadPool-7835-thread-3) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1234674 INFO (zkConnectionManagerCallback-7872-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1234676 INFO (closeThreadPool-7835-thread-2) [n:127.0.0.1:43852_mz%2Fx ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1234678 INFO (closeThreadPool-7835-thread-2) [n:127.0.0.1:43852_mz%2Fx ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44434/solr ready
[junit4] 2> 1234680 INFO (closeThreadPool-7835-thread-2) [n:127.0.0.1:43852_mz%2Fx ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1234709 INFO (closeThreadPool-7835-thread-2) [n:127.0.0.1:43852_mz%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e
[junit4] 2> 1234727 INFO (closeThreadPool-7835-thread-2) [n:127.0.0.1:43852_mz%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e
[junit4] 2> 1234727 INFO (closeThreadPool-7835-thread-2) [n:127.0.0.1:43852_mz%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e
[junit4] 2> 1234729 INFO (closeThreadPool-7835-thread-2) [n:127.0.0.1:43852_mz%2Fx ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/shard-2-001/cores
[junit4] 2> 1234793 INFO (TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/shard-4-001 of type NRT
[junit4] 2> 1234793 WARN (closeThreadPool-7835-thread-2) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 1234794 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 1234794 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 1234794 INFO (closeThreadPool-7835-thread-2) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 1234834 INFO (closeThreadPool-7835-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1234834 INFO (closeThreadPool-7835-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1234834 INFO (closeThreadPool-7835-thread-2) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1234835 INFO (closeThreadPool-7835-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@295ed2a2{/mz/x,null,AVAILABLE}
[junit4] 2> 1234835 INFO (closeThreadPool-7835-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1c30732c{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:38233}
[junit4] 2> 1234835 INFO (closeThreadPool-7835-thread-2) [ ] o.e.j.s.Server Started @1234888ms
[junit4] 2> 1234835 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:38559/hdfs__localhost_38559__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-BadApples-NightlyTests-master_checkout_solr_build_solr-core_test_J2_temp_solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001_tempDir-002_jetty4, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/mz/x, hostPort=38233, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/shard-4-001/cores}
[junit4] 2> 1234836 ERROR (closeThreadPool-7835-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1234836 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1234836 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 1234836 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1234836 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1234836 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-02-23T15:58:44.226Z
[junit4] 2> 1234845 INFO (zkConnectionManagerCallback-7875-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1234850 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1234850 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest_29E266F0FBEBE94-001/shard-4-001/solr.xml
[junit4] 2> 1234855 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1234855 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1234857 INFO (closeThreadPool-7835-thread-2) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1234870 INFO (closeThreadPool-7835-thread-3) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 1234872 WARN (closeThreadPool-7835-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@15eafbda[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1234876 WARN (closeThreadPool-7835-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@777836a6[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1234877 INFO (closeThreadPool-7835-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44434/solr
[junit4] 2> 1234879 INFO (zkConnectionManagerCallback-7881-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1234881 INFO (zkConnectionManagerCallback-7883-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1234886 INFO (closeThreadPool-7835-thread-3) [n:127.0.0.1:44073_mz%2Fx ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1234890 INFO (closeThreadPool-7835-thread-3) [n:127.0.0.1:44073_mz%2Fx ] o.a.s.c.ZkController Publish node=127.0.0.1:44073_mz%2Fx as DOWN
[junit4] 2> 1234892 INFO (closeThreadPool-7835-thread-3) [n:127.0.0.1:44073_mz%2Fx ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1234892 INFO (closeThreadPool-7835-thread-3) [n:127.0.0.1:44073_mz%2Fx ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44073_mz%2Fx
[junit4] 2> 1234894 INFO (zkCallback-7846-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1234894 INFO (zkCallback-7833-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1234894 INFO (zkCallback-7821-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1234894 INFO (zkCallback-7814-thread-3) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1234904 INFO (zkCallback-7853-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1234904 INFO (zkCallback-7871-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1234904 INFO (zkCallback-7862-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1234905 INFO (zkCallback-7882-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1234966 INFO (zkConnectionManagerCallback-7890-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1234967 INFO (closeThreadPool-7835-thread-3) [n:127.0.0.1:44073_mz%2Fx ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 1234969 INFO (closeThreadPool-7835-thread-3) [n:127.0.0.1:44073_mz%2Fx ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44434/solr ready
[junit4] 2> 1234973 INFO (closeThreadPool-7835-thread-3) [n:127.0.0.1:44073_mz%2Fx ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1235002 INFO (closeThreadPool-7835-thread-3) [n:127.0.0.1:44073_mz%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e
[junit4] 2> 1235017 INFO (closeThreadPool-7835-thread-3) [n:127.0.0.1:44073_mz%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e
[junit4] 2> 1235017 INFO (closeThreadPool-7835-thread-3) [n:127.0.0.1:44073_mz%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3adb243e
[junit4] 2> 1235019 INFO (closeThreadPool-7835-thread-3) [n:127.0.0.1:44073_mz%2Fx ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/sol
[...truncated too long message...]
ava:748)
[junit4] 2> 2325393 INFO (TEST-StressHdfsTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:38111
[junit4] 2> 2325393 INFO (TEST-StressHdfsTest.test-seed#[29E266F0FBEBE94]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 38111
[junit4] 2> 2325400 WARN (SUITE-StressHdfsTest-seed#[29E266F0FBEBE94]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 2325554 INFO (SUITE-StressHdfsTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.w.WebAppContext@78dd6717{datanode,/,null,UNAVAILABLE}{/datanode}
[junit4] 2> 2325554 INFO (SUITE-StressHdfsTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@2746a3e3{HTTP/1.1,[http/1.1]}{localhost:0}
[junit4] 2> 2325555 INFO (SUITE-StressHdfsTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.session node0 Stopped scavenging
[junit4] 2> 2325555 INFO (SUITE-StressHdfsTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2825f12e{static,/static,jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,UNAVAILABLE}
[junit4] 2> 2325558 WARN (BP-110225281-10.41.0.5-1550938479821 heartbeating to localhost/127.0.0.1:41833) [ ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
[junit4] 2> 2325558 WARN (BP-110225281-10.41.0.5-1550938479821 heartbeating to localhost/127.0.0.1:41833) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-110225281-10.41.0.5-1550938479821 (Datanode Uuid fa805903-c767-4f13-969d-8d8ea44025e8) service to localhost/127.0.0.1:41833
[junit4] 2> 2325576 WARN (refreshUsed-/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.StressHdfsTest_29E266F0FBEBE94-001/tempDir-001/hdfsBaseDir/data/data3/current/BP-110225281-10.41.0.5-1550938479821) [ ] o.a.h.f.CachingGetSpaceUsed Thread Interrupted waiting to refresh disk information: sleep interrupted
[junit4] 2> 2325637 WARN (refreshUsed-/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.StressHdfsTest_29E266F0FBEBE94-001/tempDir-001/hdfsBaseDir/data/data4/current/BP-110225281-10.41.0.5-1550938479821) [ ] o.a.h.f.CachingGetSpaceUsed Thread Interrupted waiting to refresh disk information: sleep interrupted
[junit4] 2> 2325646 WARN (SUITE-StressHdfsTest-seed#[29E266F0FBEBE94]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 2325786 INFO (SUITE-StressHdfsTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.w.WebAppContext@47bd7c13{datanode,/,null,UNAVAILABLE}{/datanode}
[junit4] 2> 2325787 INFO (SUITE-StressHdfsTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@18f8378f{HTTP/1.1,[http/1.1]}{localhost:0}
[junit4] 2> 2325787 INFO (SUITE-StressHdfsTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.session node0 Stopped scavenging
[junit4] 2> 2325787 INFO (SUITE-StressHdfsTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@35ec66f4{static,/static,jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,UNAVAILABLE}
[junit4] 2> 2325789 WARN (BP-110225281-10.41.0.5-1550938479821 heartbeating to localhost/127.0.0.1:41833) [ ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
[junit4] 2> 2325789 WARN (BP-110225281-10.41.0.5-1550938479821 heartbeating to localhost/127.0.0.1:41833) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-110225281-10.41.0.5-1550938479821 (Datanode Uuid 42897d9a-7567-49fb-b3f6-28484defb9b6) service to localhost/127.0.0.1:41833
[junit4] 2> 2325802 WARN (refreshUsed-/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.StressHdfsTest_29E266F0FBEBE94-001/tempDir-001/hdfsBaseDir/data/data1/current/BP-110225281-10.41.0.5-1550938479821) [ ] o.a.h.f.CachingGetSpaceUsed Thread Interrupted waiting to refresh disk information: sleep interrupted
[junit4] 2> 2325830 WARN (refreshUsed-/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.StressHdfsTest_29E266F0FBEBE94-001/tempDir-001/hdfsBaseDir/data/data2/current/BP-110225281-10.41.0.5-1550938479821) [ ] o.a.h.f.CachingGetSpaceUsed Thread Interrupted waiting to refresh disk information: sleep interrupted
[junit4] 2> 2325864 INFO (SUITE-StressHdfsTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.w.WebAppContext@358afb7{hdfs,/,null,UNAVAILABLE}{/hdfs}
[junit4] 2> 2325869 INFO (SUITE-StressHdfsTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@5dff9c94{HTTP/1.1,[http/1.1]}{localhost:0}
[junit4] 2> 2325869 INFO (SUITE-StressHdfsTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.session node0 Stopped scavenging
[junit4] 2> 2325870 INFO (SUITE-StressHdfsTest-seed#[29E266F0FBEBE94]-worker) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@32343883{static,/static,jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,UNAVAILABLE}
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.StressHdfsTest_29E266F0FBEBE94-001
[junit4] 2> Feb 23, 2019 4:16:56 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 32 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene80), sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@be7b5fb), locale=zh, timezone=America/Cambridge_Bay
[junit4] 2> NOTE: Linux 4.4.0-137-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=4,free=133881712,total=473956352
[junit4] 2> NOTE: All tests run in this JVM: [JvmMetricsTest, TestScoreJoinQPNoScore, BasicFunctionalityTest, MultiSolrCloudTestCaseTest, ExplicitHLLTest, DistributedUpdateProcessorTest, TestDocTermOrds, TestTolerantSearch, TestHighlightDedupGrouping, ZkSolrClientTest, TestCloudJSONFacetSKG, TestPartialUpdateDeduplication, TestSimpleQParserPlugin, TestCursorMarkWithoutUniqueKey, SchemaVersionSpecificBehaviorTest, CdcrWithNodesRestartsTest, RequestHandlersTest, TestRestoreCore, IndexSizeTriggerTest, TestSolrXml, TestPayloadScoreQParserPlugin, FullSolrCloudDistribCmdsTest, SolrJmxReporterTest, PreAnalyzedUpdateProcessorTest, OverseerModifyCollectionTest, ExitableDirectoryReaderTest, TestDynamicFieldResource, TlogReplayBufferedWhileIndexingTest, URLClassifyProcessorTest, LoggingHandlerTest, UtilsToolTest, CollectionPropsTest, RankQueryTest, MetricsHistoryIntegrationTest, TestNoOpRegenerator, RecoveryAfterSoftCommitTest, MoveReplicaHDFSFailoverTest, JsonLoaderTest, TestUtilizeNode, SuggesterTest, TestCorePropertiesReload, ZkStateWriterTest, HLLUtilTest, CloudExitableDirectoryReaderTest, TestFoldingMultitermQuery, TestSimPolicyCloud, TestRandomFaceting, TestNumericTerms32, UUIDFieldTest, SimpleFacetsTest, TestReloadDeadlock, TestFieldCacheVsDocValues, RootFieldTest, PeerSyncWithIndexFingerprintCachingTest, TestCollapseQParserPlugin, SolrGangliaReporterTest, TestSimExecutePlanAction, TestCloudManagedSchema, JavabinLoaderTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestComponentsName, TestFaceting, HdfsBasicDistributedZkTest, CreateCollectionCleanupTest, SortSpecParsingTest, DistributedFacetPivotWhiteBoxTest, DistributedFacetPivotLongTailTest, TestPerFieldSimilarityWithDefaultOverride, TestEmbeddedSolrServerConstructors, BufferStoreTest, DistribDocExpirationUpdateProcessorTest, DeleteLastCustomShardedReplicaTest, TestPointFields, TestSimLargeCluster, ProtectedTermFilterFactoryTest, PrimitiveFieldTypeTest, SolrShardReporterTest, TestSolr4Spatial2, BlockJoinFacetDistribTest, SolrInfoBeanTest, LeaderElectionTest, ConcurrentDeleteAndCreateCollectionTest, TestCloudRecovery2, TestLegacyBM25SimilarityFactory, CopyFieldTest, TestCryptoKeys, StressHdfsTest]
[junit4] Completed [351/844 (2!)] on J1 in 138.30s, 1 test, 1 error <<< FAILURES!
[...truncated 1632 lines...]
[junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20190223_153809_0964861013134496910624.sysout
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/heapdumps/java_pid15365.hprof ...
[junit4] Heap dump file created [616841767 bytes in 41.047 secs]
[junit4] <<< JVM J1: EOF ----
[...truncated 8929 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/build.xml:661: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid15365.hprof
Total time: 298 minutes 27 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-BadApples-NightlyTests-master - Build # 50 -
Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-master/50/
5 tests failed.
FAILED: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test
Error Message:
Test abandoned because suite timeout was reached.
Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
at __randomizedtesting.SeedInfo.seed([78826EB40D057D8C]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([78826EB40D057D8C]:0)
FAILED: org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey
Error Message:
Address already in use
Stack Trace:
java.net.BindException: Address already in use
at __randomizedtesting.SeedInfo.seed([78826EB40D057D8C:F3A5BD654C03D608]:0)
at sun.nio.ch.Net.bind0(Native Method)
at sun.nio.ch.Net.bind(Net.java:433)
at sun.nio.ch.Net.bind(Net.java:425)
at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
at org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:342)
at org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:308)
at org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
at org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:236)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.eclipse.jetty.server.Server.doStart(Server.java:394)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:544)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:483)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:451)
at org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:499)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest
Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest: 1) Thread[id=166070, name=qtp1647625371-166070, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/1225342635.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=166078, name=Scheduler-1296012406, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=166073, name=qtp1647625371-166073, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=166134, name=Scheduler-697434102, state=WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=166072, name=qtp1647625371-166072-acceptor-0@6a97452b-ServerConnector@2a16db62{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:43285}, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=166075, name=qtp1647625371-166075, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=166074, name=qtp1647625371-166074, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=166071, name=qtp1647625371-166071, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/1225342635.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=166076, name=qtp1647625371-166076, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=166077, name=qtp1647625371-166077, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest:
1) Thread[id=166070, name=qtp1647625371-166070, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/1225342635.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=166078, name=Scheduler-1296012406, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=166073, name=qtp1647625371-166073, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=166134, name=Scheduler-697434102, state=WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=166072, name=qtp1647625371-166072-acceptor-0@6a97452b-ServerConnector@2a16db62{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:43285}, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=166075, name=qtp1647625371-166075, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=166074, name=qtp1647625371-166074, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=166071, name=qtp1647625371-166071, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/1225342635.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=166076, name=qtp1647625371-166076, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=166077, name=qtp1647625371-166077, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([78826EB40D057D8C]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=166070, name=qtp1647625371-166070, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/1225342635.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=166078, name=Scheduler-1296012406, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=166071, name=qtp1647625371-166071, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/1225342635.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=166134, name=Scheduler-697434102, state=WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=166070, name=qtp1647625371-166070, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/1225342635.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=166078, name=Scheduler-1296012406, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=166071, name=qtp1647625371-166071, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/1225342635.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=166134, name=Scheduler-697434102, state=WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([78826EB40D057D8C]:0)
Build Log:
[...truncated 14871 lines...]
[junit4] Suite: org.apache.solr.cloud.api.collections.ShardSplitTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_78826EB40D057D8C-001/init-core-data-001
[junit4] 2> 4093392 WARN (SUITE-ShardSplitTest-seed#[78826EB40D057D8C]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=96 numCloses=96
[junit4] 2> 4093392 INFO (SUITE-ShardSplitTest-seed#[78826EB40D057D8C]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 4093394 INFO (SUITE-ShardSplitTest-seed#[78826EB40D057D8C]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 4093394 INFO (SUITE-ShardSplitTest-seed#[78826EB40D057D8C]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 4093395 INFO (SUITE-ShardSplitTest-seed#[78826EB40D057D8C]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 4093402 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 4093402 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 4093402 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 4093502 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.ZkTestServer start zk server on port:40227
[junit4] 2> 4093502 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:40227
[junit4] 2> 4093502 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 40227
[junit4] 2> 4093509 INFO (zkConnectionManagerCallback-74827-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4093516 INFO (zkConnectionManagerCallback-74829-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4093517 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 4093518 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 4093520 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 4093521 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 4093522 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 4093523 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 4093524 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 4093525 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 4093527 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 4093528 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 4093529 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 4093530 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 4093701 WARN (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 4093702 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 4093702 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 4093702 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 4093703 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 4093703 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 4093703 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 4093703 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6b11247{/,null,AVAILABLE}
[junit4] 2> 4093706 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@70f934f4{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:38518}
[junit4] 2> 4093706 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.e.j.s.Server Started @4093763ms
[junit4] 2> 4093706 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_78826EB40D057D8C-001/tempDir-001/control/data, replicaType=NRT, hostContext=/, hostPort=38518, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_78826EB40D057D8C-001/control-001/cores}
[junit4] 2> 4093706 ERROR (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4093706 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 4093707 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 4093707 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4093707 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 4093707 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-02-16T05:31:38.449Z
[junit4] 2> 4093708 INFO (zkConnectionManagerCallback-74831-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4093710 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 4093710 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_78826EB40D057D8C-001/control-001/solr.xml
[junit4] 2> 4093715 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 4093715 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 4093717 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@599e676, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 4093858 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 4093860 WARN (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@2071096e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 4093865 WARN (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@41db5ac2[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 4093866 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40227/solr
[junit4] 2> 4093867 INFO (zkConnectionManagerCallback-74837-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4093870 INFO (zkConnectionManagerCallback-74839-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4093976 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38518_
[junit4] 2> 4093977 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [n:127.0.0.1:38518_ ] o.a.s.c.Overseer Overseer (id=72806661248909316-127.0.0.1:38518_-n_0000000000) starting
[junit4] 2> 4093988 INFO (zkConnectionManagerCallback-74846-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4093991 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [n:127.0.0.1:38518_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40227/solr ready
[junit4] 2> 4093992 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [n:127.0.0.1:38518_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38518_
[junit4] 2> 4093992 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 4093992 INFO (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:38518_
[junit4] 2> 4093993 DEBUG (OverseerAutoScalingTriggerThread-72806661248909316-127.0.0.1:38518_-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 4093994 INFO (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4093994 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0 #Completed tasks: 0
[junit4] 2> 4093994 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 4093994 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 4093994 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 4093995 INFO (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
[junit4] 2> 4093995 INFO (zkCallback-74845-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4093996 DEBUG (OverseerAutoScalingTriggerThread-72806661248909316-127.0.0.1:38518_-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 4093997 DEBUG (OverseerAutoScalingTriggerThread-72806661248909316-127.0.0.1:38518_-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 4093997 DEBUG (OverseerAutoScalingTriggerThread-72806661248909316-127.0.0.1:38518_-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 4094017 DEBUG (OverseerAutoScalingTriggerThread-72806661248909316-127.0.0.1:38518_-n_0000000000) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:38518_]
[junit4] 2> 4094017 DEBUG (OverseerAutoScalingTriggerThread-72806661248909316-127.0.0.1:38518_-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 4094018 DEBUG (OverseerAutoScalingTriggerThread-72806661248909316-127.0.0.1:38518_-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 4094018 DEBUG (ScheduledTrigger-21556-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 4094026 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [n:127.0.0.1:38518_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 4094055 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [n:127.0.0.1:38518_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@599e676
[junit4] 2> 4094071 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [n:127.0.0.1:38518_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@599e676
[junit4] 2> 4094071 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [n:127.0.0.1:38518_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@599e676
[junit4] 2> 4094073 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [n:127.0.0.1:38518_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_78826EB40D057D8C-001/control-001/cores
[junit4] 2> 4094121 INFO (zkConnectionManagerCallback-74852-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4094127 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4094128 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40227/solr ready
[junit4] 2> 4094129 INFO (qtp1480237369-165081) [n:127.0.0.1:38518_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38518_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4094137 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Got 1 tasks from work-queue : [[org.apache.solr.cloud.OverseerTaskQueue$QueueEvent@2e684ecf]]
[junit4] 2> 4094137 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as running
[junit4] 2> 4094137 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
[junit4] 2> "name":"control_collection",
[junit4] 2> "fromApi":"true",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"1",
[junit4] 2> "createNodeSet":"127.0.0.1:38518_",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "operation":"create"}
[junit4] 2> 4094137 DEBUG (OverseerThreadFactory-21558-thread-1-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Runner processing /overseer/collection-queue-work/qn-0000000000
[junit4] 2> 4094138 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1 #Completed tasks: 0
[junit4] 2> 4094138 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 4094138 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 4094138 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 4094138 INFO (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 4094138 DEBUG (OverseerThreadFactory-21558-thread-1-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.a.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
[junit4] 2> "name":"control_collection",
[junit4] 2> "fromApi":"true",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"1",
[junit4] 2> "createNodeSet":"127.0.0.1:38518_",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "operation":"create"}
[junit4] 2> 4094138 INFO (OverseerThreadFactory-21558-thread-1-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 4094139 DEBUG (OverseerThreadFactory-21558-thread-1-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.a.c.OverseerCollectionMessageHandler creating collections conf node /collections/control_collection
[junit4] 2> 4094139 DEBUG (OverseerThreadFactory-21558-thread-1-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.a.c.CreateCollectionCmd Check for collection zkNode:control_collection
[junit4] 2> 4094140 DEBUG (OverseerThreadFactory-21558-thread-1-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.a.c.CreateCollectionCmd Collection zkNode exists
[junit4] 2> 4094143 DEBUG (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "name":"control_collection",
[junit4] 2> "fromApi":"true",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"1",
[junit4] 2> "createNodeSet":"127.0.0.1:38518_",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "operation":"create"} current state version: 0
[junit4] 2> 4094143 DEBUG (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.o.ClusterStateMutator building a new cName: control_collection
[junit4] 2> 4094144 DEBUG (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/control_collection/state.json
[junit4] 2> 4094242 DEBUG (OverseerThreadFactory-21558-thread-1-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.a.c.Assign shardnames [shard1] NRT 1 TLOG 0 PULL 0 , policy null, nodeList [127.0.0.1:38518_]
[junit4] 2> 4094245 INFO (qtp1480237369-165083) [n:127.0.0.1:38518_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 4094246 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 4094247 INFO (qtp1480237369-165080) [n:127.0.0.1:38518_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 4094248 INFO (qtp1480237369-165082) [n:127.0.0.1:38518_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 4094248 INFO (qtp1480237369-165083) [n:127.0.0.1:38518_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 4094252 DEBUG (OverseerThreadFactory-21558-thread-1-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.a.c.CreateCollectionCmd Creating SolrCores for new collection control_collection, shardNames [shard1] , message : {
[junit4] 2> "name":"control_collection",
[junit4] 2> "fromApi":"true",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"1",
[junit4] 2> "createNodeSet":"127.0.0.1:38518_",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "operation":"create"}
[junit4] 2> 4094253 DEBUG (OverseerThreadFactory-21558-thread-1-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.a.c.CreateCollectionCmd Creating core control_collection_shard1_replica_n1 as part of shard shard1 of collection control_collection on 127.0.0.1:38518_
[junit4] 2> 4094259 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 4094260 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 4094264 DEBUG (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "core":"control_collection_shard1_replica_n1",
[junit4] 2> "roles":null,
[junit4] 2> "base_url":"http://127.0.0.1:38518",
[junit4] 2> "node_name":"127.0.0.1:38518_",
[junit4] 2> "numShards":"1",
[junit4] 2> "state":"down",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"control_collection",
[junit4] 2> "type":"NRT",
[junit4] 2> "operation":"state"} current state version: 0
[junit4] 2> 4094264 DEBUG (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
[junit4] 2> "core":"control_collection_shard1_replica_n1",
[junit4] 2> "roles":null,
[junit4] 2> "base_url":"http://127.0.0.1:38518",
[junit4] 2> "node_name":"127.0.0.1:38518_",
[junit4] 2> "numShards":"1",
[junit4] 2> "state":"down",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"control_collection",
[junit4] 2> "type":"NRT",
[junit4] 2> "operation":"state"}
[junit4] 2> 4094265 DEBUG (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.o.ReplicaMutator Will update state for replica: core_node2:{"core":"control_collection_shard1_replica_n1","base_url":"http://127.0.0.1:38518","node_name":"127.0.0.1:38518_","state":"down","type":"NRT"}
[junit4] 2> 4094265 DEBUG (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.o.ReplicaMutator Collection is now: DocCollection(control_collection//collections/control_collection/state.json/0)={
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active",
[junit4] 2> "replicas":{"core_node2":{
[junit4] 2> "core":"control_collection_shard1_replica_n1",
[junit4] 2> "base_url":"http://127.0.0.1:38518",
[junit4] 2> "node_name":"127.0.0.1:38518_",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT"}}}}}
[junit4] 2> 4094366 DEBUG (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/control_collection/state.json version: 0
[junit4] 2> 4095023 DEBUG (ScheduledTrigger-21556-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 4095295 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 4095311 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 4095422 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 4095440 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 4095441 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@599e676
[junit4] 2> 4095442 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_78826EB40D057D8C-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.ShardSplitTest_78826EB40D057D8C-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 4095445 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@1bdeb758
[junit4] 2> 4095452 WARN (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 4095515 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 4095515 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 4095519 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 4095519 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 4095520 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=1.15625, floorSegmentMB=1.625, forceMergeDeletesPctAllowed=12.61510136687462, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0, deletesPctAllowed=27.652961322269665
[junit4] 2> 4095521 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@67496f40[control_collection_shard1_replica_n1] main]
[junit4] 2> 4095522 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 4095523 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 4095523 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 4095523 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1625602235055472640
[junit4] 2> 4095528 INFO (searcherExecutor-21563-thread-1-processing-n:127.0.0.1:38518_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@67496f40[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 4095531 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 4095532 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
[junit4] 2> 4095535 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 4095535 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 4095535 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:38518/control_collection_shard1_replica_n1/
[junit4] 2> 4095536 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 4095536 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:38518/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 4095536 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72806661248909316-core_node2-n_0000000000
[junit4] 2> 4095538 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:38518/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 4095539 DEBUG (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"leader",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"control_collection",
[junit4] 2> "base_url":"http://127.0.0.1:38518",
[junit4] 2> "core":"control_collection_shard1_replica_n1",
[junit4] 2> "state":"active"} current state version: 0
[junit4] 2> 4095639 DEBUG (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/control_collection/state.json version: 1
[junit4] 2> 4095640 INFO (zkCallback-74838-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4095640 INFO (zkCallback-74838-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4095641 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 4095642 DEBUG (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "core":"control_collection_shard1_replica_n1",
[junit4] 2> "core_node_name":"core_node2",
[junit4] 2> "roles":null,
[junit4] 2> "base_url":"http://127.0.0.1:38518",
[junit4] 2> "node_name":"127.0.0.1:38518_",
[junit4] 2> "numShards":"1",
[junit4] 2> "state":"active",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"control_collection",
[junit4] 2> "type":"NRT",
[junit4] 2> "operation":"state"} current state version: 0
[junit4] 2> 4095642 DEBUG (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
[junit4] 2> "core":"control_collection_shard1_replica_n1",
[junit4] 2> "core_node_name":"core_node2",
[junit4] 2> "roles":null,
[junit4] 2> "base_url":"http://127.0.0.1:38518",
[junit4] 2> "node_name":"127.0.0.1:38518_",
[junit4] 2> "numShards":"1",
[junit4] 2> "state":"active",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"control_collection",
[junit4] 2> "type":"NRT",
[junit4] 2> "operation":"state"}
[junit4] 2> 4095642 DEBUG (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.o.ReplicaMutator Will update state for replica: core_node2:{"core":"control_collection_shard1_replica_n1","base_url":"http://127.0.0.1:38518","node_name":"127.0.0.1:38518_","state":"active","type":"NRT","leader":"true"}
[junit4] 2> 4095642 DEBUG (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.o.ReplicaMutator Collection is now: DocCollection(control_collection//collections/control_collection/state.json/2)={
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active",
[junit4] 2> "replicas":{"core_node2":{
[junit4] 2> "core":"control_collection_shard1_replica_n1",
[junit4] 2> "base_url":"http://127.0.0.1:38518",
[junit4] 2> "node_name":"127.0.0.1:38518_",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "leader":"true"}}}}}
[junit4] 2> 4095643 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1384
[junit4] 2> 4095643 DEBUG (OverseerThreadFactory-21558-thread-1-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.a.c.CreateCollectionCmd Finished create command on all shards for collection: control_collection
[junit4] 2> 4095643 DEBUG (OverseerThreadFactory-21558-thread-1-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Completed task:[/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 4095644 DEBUG (OverseerThreadFactory-21558-thread-1-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as completed.
[junit4] 2> 4095644 DEBUG (OverseerThreadFactory-21558-thread-1-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 4095645 DEBUG (OverseerThreadFactory-21558-thread-1-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 4095645 DEBUG (OverseerThreadFactory-21558-thread-1-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor CompletedTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 4095645 INFO (OverseerThreadFactory-21558-thread-1-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 4095645 DEBUG (OverseerThreadFactory-21558-thread-1-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Message id:/overseer/collection-queue-work/qn-0000000000 complete, response:{success={127.0.0.1:38518_={responseHeader={status=0,QTime=1384},core=control_collection_shard1_replica_n1}}}
[junit4] 2> 4095648 INFO (qtp1480237369-165081) [n:127.0.0.1:38518_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 4095743 DEBUG (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/control_collection/state.json version: 2
[junit4] 2> 4095743 INFO (zkCallback-74838-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4095743 INFO (zkCallback-74838-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4095743 INFO (zkCallback-74838-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4095744 INFO (qtp1480237369-165081) [n:127.0.0.1:38518_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38518_&wt=javabin&version=2} status=0 QTime=1614
[junit4] 2> 4095750 INFO (zkConnectionManagerCallback-74858-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4095751 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4095753 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40227/solr ready
[junit4] 2> 4095753 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[78826EB40D057D8C]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 4095754 INFO (qtp1480237369-165084) [n:127.0.0.1:38518_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4095756 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Got 1 tasks from work-queue : [[org.apache.solr.cloud.OverseerTaskQueue$QueueEvent@2e684ed1]]
[junit4] 2> 4095756 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000002] as running
[junit4] 2> 4095756 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000002 message:{
[junit4] 2> "name":"collection1",
[junit4] 2> "fromApi":"true",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"2",
[junit4] 2> "createNodeSet":"",
[junit4] 2> "stateFormat":"1",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "operation":"create"}
[junit4] 2> 4095757 DEBUG (OverseerThreadFactory-21558-thread-2-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Runner processing /overseer/collection-queue-work/qn-0000000002
[junit4] 2> 4095757 DEBUG (OverseerThreadFactory-21558-thread-2-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.a.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
[junit4] 2> "name":"collection1",
[junit4] 2> "fromApi":"true",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"2",
[junit4] 2> "createNodeSet":"",
[junit4] 2> "stateFormat":"1",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "operation":"create"}
[junit4] 2> 4095757 INFO (OverseerThreadFactory-21558-thread-2-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 4095757 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1 #Completed tasks: 1
[junit4] 2> 4095757 INFO (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 4095757 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000002]
[junit4] 2> 4095757 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 4095757 DEBUG (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 4095757 INFO (OverseerCollectionConfigSetProcessor-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000002]
[junit4] 2> 4095758 DEBUG (OverseerThreadFactory-21558-thread-2-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.a.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection1
[junit4] 2> 4095758 DEBUG (OverseerThreadFactory-21558-thread-2-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.a.c.CreateCollectionCmd Check for collection zkNode:collection1
[junit4] 2> 4095759 DEBUG (OverseerThreadFactory-21558-thread-2-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.a.c.CreateCollectionCmd Collection zkNode exists
[junit4] 2> 4095761 DEBUG (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "name":"collection1",
[junit4] 2> "fromApi":"true",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"2",
[junit4] 2> "createNodeSet":"",
[junit4] 2> "stateFormat":"1",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "operation":"create"} current state version: 0
[junit4] 2> 4095761 DEBUG (OverseerStateUpdate-72806661248909316-127.0.0.1:38518_-n_0000000000) [n:127.0.0.1:38518_ ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
[junit4] 2> 4095961 WARN (OverseerThreadFactory-21558-thread-2-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 4095961 DEBUG (OverseerThreadFactory-21558-thread-2-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.a.c.CreateCollectionCmd Finished create command for collection: collection1
[junit4] 2> 4095961 DEBUG (OverseerThreadFactory-21558-thread-2-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Completed task:[/overseer/collection-queue-work/qn-0000000002]
[junit4] 2> 4095963 DEBUG (OverseerThreadFactory-21558-thread-2-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000002] as completed.
[junit4] 2> 4095963 DEBUG (OverseerThreadFactory-21558-thread-2-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 4095963 DEBUG (OverseerThreadFactory-21558-thread-2-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 4095963 DEBUG (OverseerThreadFactory-21558-thread-2-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor CompletedTasks: [/overseer/collection-queue-work/qn-0000000002]
[junit4] 2> 4095963 INFO (OverseerThreadFactory-21558-thread-2-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000002]
[junit4] 2> 4095963 DEBUG (OverseerThreadFactory-21558-thread-2-processing-n:127.0.0.1:38518_) [n:127.0.0.1:38518_ ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Message id:/overseer/collection-queue-work/qn-0000000002 complete, response:{}
[junit4] 2> 40
[...truncated too long message...]
ile = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null943724569
[copy] Copying 237 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null943724569
[delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null943724569
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 54ms :: artifacts dl 4ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 408 minutes 28 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2847)
at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2726)
at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2707)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3086)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1072)
at hudson.FilePath.act(FilePath.java:1061)
at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1835)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3088)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1074)
at hudson.FilePath.act(FilePath.java:1061)
at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1835)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
[JENKINS] Lucene-Solr-BadApples-NightlyTests-master - Build # 49 -
Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-master/49/
2 tests failed.
FAILED: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test
Error Message:
Captured an uncaught exception in thread: Thread[id=54311, name=Thread-36791, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=54311, name=Thread-36791, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:46237/collection2_shard4_replica_n17
at __randomizedtesting.SeedInfo.seed([386ED1F1C26D0A10]:0)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:638)
Caused by: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:46237/collection2_shard4_replica_n17
at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:567)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1019)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:224)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:635)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:46237/collection2_shard4_replica_n17
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:661)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:256)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:245)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:213)
at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:561)
... 6 more
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:549)
... 12 more
FAILED: junit.framework.TestSuite.org.apache.solr.client.solrj.impl.CloudSolrClientTest
Error Message:
ObjectTracker found 2 object(s) that were not released!!! [RawDirectoryWrapper, SolrCore] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:778) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:975) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:882) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1099) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:188) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1062) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:882) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1099) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:188) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) expected null, but was:<ObjectTracker found 2 object(s) that were not released!!! [RawDirectoryWrapper, SolrCore] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:778) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:975) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:882) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1099) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:188) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1062) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:882) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1099) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:188) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) >
Stack Trace:
java.lang.AssertionError: ObjectTracker found 2 object(s) that were not released!!! [RawDirectoryWrapper, SolrCore]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)
at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:778)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:975)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:882)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1099)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)
at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:188)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1062)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:882)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1099)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)
at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:188)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
expected null, but was:<ObjectTracker found 2 object(s) that were not released!!! [RawDirectoryWrapper, SolrCore]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)
at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:778)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:975)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:882)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1099)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)
at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:188)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1062)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:882)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1189)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1099)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)
at org.apache.solr.handler.admin.CoreAdminHandler.lambda$handleRequestBody$0(CoreAdminHandler.java:188)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
>
at __randomizedtesting.SeedInfo.seed([69DCD5F2E716A9AF]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.failNotNull(Assert.java:755)
at org.junit.Assert.assertNull(Assert.java:737)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:322)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:901)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 15510 lines...]
[junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/init-core-data-001
[junit4] 2> 2314884 WARN (SUITE-FullSolrCloudDistribCmdsTest-seed#[386ED1F1C26D0A10]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=9 numCloses=9
[junit4] 2> 2314884 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[386ED1F1C26D0A10]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 2314887 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[386ED1F1C26D0A10]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 2314887 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[386ED1F1C26D0A10]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 2314887 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[386ED1F1C26D0A10]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 2314892 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2314893 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2314893 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2314993 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.ZkTestServer start zk server on port:39082
[junit4] 2> 2314993 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:39082
[junit4] 2> 2314993 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 39082
[junit4] 2> 2315002 INFO (zkConnectionManagerCallback-7089-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2315005 INFO (zkConnectionManagerCallback-7091-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2315006 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 2315007 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 2315009 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 2315010 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 2315011 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 2315012 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 2315013 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 2315014 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 2315016 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 2315017 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 2315018 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 2315019 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 2315200 WARN (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 2315200 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 2315200 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 2315200 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 2315201 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2315201 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2315201 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 2315202 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@71ff6775{/,null,AVAILABLE}
[junit4] 2> 2315205 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@576216a3{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:45644}
[junit4] 2> 2315205 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.e.j.s.Server Started @2315258ms
[junit4] 2> 2315205 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/tempDir-001/control/data, hostContext=/, hostPort=45644, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/control-001/cores}
[junit4] 2> 2315205 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2315205 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 2315206 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 2315206 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2315206 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2315206 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-02-09T16:16:03.898Z
[junit4] 2> 2315208 INFO (zkConnectionManagerCallback-7093-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2315210 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2315210 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/control-001/solr.xml
[junit4] 2> 2315215 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2315215 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2315217 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2315296 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 2315298 WARN (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@3af21c89[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2315303 WARN (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@1d11d93d[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2315304 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39082/solr
[junit4] 2> 2315305 INFO (zkConnectionManagerCallback-7099-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2315308 INFO (zkConnectionManagerCallback-7101-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2315399 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [n:127.0.0.1:45644_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45644_
[junit4] 2> 2315400 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [n:127.0.0.1:45644_ ] o.a.s.c.Overseer Overseer (id=72769559088201732-127.0.0.1:45644_-n_0000000000) starting
[junit4] 2> 2315408 INFO (zkConnectionManagerCallback-7108-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2315412 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [n:127.0.0.1:45644_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39082/solr ready
[junit4] 2> 2315412 INFO (OverseerStateUpdate-72769559088201732-127.0.0.1:45644_-n_0000000000) [n:127.0.0.1:45644_ ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:45644_
[junit4] 2> 2315413 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [n:127.0.0.1:45644_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45644_
[junit4] 2> 2315415 INFO (zkCallback-7100-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2315416 INFO (zkCallback-7107-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2315470 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [n:127.0.0.1:45644_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 2315499 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [n:127.0.0.1:45644_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635
[junit4] 2> 2315514 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [n:127.0.0.1:45644_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635
[junit4] 2> 2315514 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [n:127.0.0.1:45644_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635
[junit4] 2> 2315516 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [n:127.0.0.1:45644_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/control-001/cores
[junit4] 2> 2315546 INFO (zkConnectionManagerCallback-7114-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2315547 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2315549 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39082/solr ready
[junit4] 2> 2315551 INFO (qtp1809927645-53641) [n:127.0.0.1:45644_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45644_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2315555 INFO (OverseerThreadFactory-5666-thread-1-processing-n:127.0.0.1:45644_) [n:127.0.0.1:45644_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 2315662 INFO (qtp1809927645-53643) [n:127.0.0.1:45644_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2315663 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2315663 INFO (qtp1809927645-53640) [n:127.0.0.1:45644_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2315664 INFO (qtp1809927645-53642) [n:127.0.0.1:45644_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2315665 INFO (qtp1809927645-53643) [n:127.0.0.1:45644_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2315668 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2315669 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2316690 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 2316706 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 2316832 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 2316852 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 2316852 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635
[junit4] 2> 2316853 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 2316855 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=55.625, floorSegmentMB=1.859375, forceMergeDeletesPctAllowed=8.679888982311304, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=31.2543796949949
[junit4] 2> 2316858 WARN (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2316919 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2316919 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2316922 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2316922 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2316923 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=44, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24232756382124496]
[junit4] 2> 2316923 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4b9a22a8[control_collection_shard1_replica_n1] main]
[junit4] 2> 2316925 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2316925 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2316926 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2316926 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1625008599851859968
[junit4] 2> 2316929 INFO (searcherExecutor-5671-thread-1-processing-n:127.0.0.1:45644_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@4b9a22a8[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2316933 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 2316933 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
[junit4] 2> 2316937 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2316937 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2316937 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45644/control_collection_shard1_replica_n1/
[junit4] 2> 2316937 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2316937 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:45644/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 2316938 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72769559088201732-core_node2-n_0000000000
[junit4] 2> 2316940 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45644/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 2317042 INFO (zkCallback-7100-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2317042 INFO (zkCallback-7100-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2317042 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2317044 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1376
[junit4] 2> 2317050 INFO (qtp1809927645-53641) [n:127.0.0.1:45644_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 2317149 INFO (zkCallback-7100-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2317149 INFO (zkCallback-7100-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2317149 INFO (zkCallback-7100-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2317150 INFO (qtp1809927645-53641) [n:127.0.0.1:45644_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45644_&wt=javabin&version=2} status=0 QTime=1599
[junit4] 2> 2317154 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x102878747ea0006, likely client has closed socket
[junit4] 2> 2317156 INFO (zkConnectionManagerCallback-7120-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2317157 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2317158 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39082/solr ready
[junit4] 2> 2317158 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 2317159 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2317162 INFO (OverseerThreadFactory-5666-thread-2-processing-n:127.0.0.1:45644_) [n:127.0.0.1:45644_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 2317163 INFO (OverseerCollectionConfigSetProcessor-72769559088201732-127.0.0.1:45644_-n_0000000000) [n:127.0.0.1:45644_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2317367 WARN (OverseerThreadFactory-5666-thread-2-processing-n:127.0.0.1:45644_) [n:127.0.0.1:45644_ ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 2317370 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 2317371 INFO (qtp1809927645-53644) [n:127.0.0.1:45644_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=211
[junit4] 2> 2317372 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=6
[junit4] 2> 2317555 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-1-001 of type NRT
[junit4] 2> 2317556 WARN (closeThreadPool-7121-thread-1) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 2317557 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 2317557 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 2317557 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 2317558 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2317558 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2317558 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 2317558 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@776ed4d6{/,null,AVAILABLE}
[junit4] 2> 2317559 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6ce37dd5{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:37569}
[junit4] 2> 2317559 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.Server Started @2317612ms
[junit4] 2> 2317559 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=37569, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-1-001/cores}
[junit4] 2> 2317559 ERROR (closeThreadPool-7121-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2317559 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 2317559 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 2317559 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2317560 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2317560 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-02-09T16:16:06.252Z
[junit4] 2> 2317564 INFO (zkConnectionManagerCallback-7123-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2317565 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2317565 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-1-001/solr.xml
[junit4] 2> 2317570 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2317570 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2317572 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2317671 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 2317696 WARN (closeThreadPool-7121-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@352d370f[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2317701 WARN (closeThreadPool-7121-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@3746ae3f[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2317702 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39082/solr
[junit4] 2> 2317710 INFO (zkConnectionManagerCallback-7129-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2317714 INFO (zkConnectionManagerCallback-7131-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2317728 INFO (closeThreadPool-7121-thread-1) [n:127.0.0.1:37569_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2317733 INFO (closeThreadPool-7121-thread-1) [n:127.0.0.1:37569_ ] o.a.s.c.ZkController Publish node=127.0.0.1:37569_ as DOWN
[junit4] 2> 2317735 INFO (closeThreadPool-7121-thread-1) [n:127.0.0.1:37569_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2317735 INFO (closeThreadPool-7121-thread-1) [n:127.0.0.1:37569_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37569_
[junit4] 2> 2317736 INFO (zkCallback-7119-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2317736 INFO (zkCallback-7107-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2317737 INFO (zkCallback-7100-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2317740 INFO (zkCallback-7130-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2317765 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-2-001 of type NRT
[junit4] 2> 2317766 WARN (closeThreadPool-7121-thread-2) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 2317767 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 2317767 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 2317767 INFO (closeThreadPool-7121-thread-2) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 2317769 INFO (closeThreadPool-7121-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2317769 INFO (closeThreadPool-7121-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2317769 INFO (closeThreadPool-7121-thread-2) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 2317769 INFO (closeThreadPool-7121-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7bd8eb72{/,null,AVAILABLE}
[junit4] 2> 2317770 INFO (closeThreadPool-7121-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2419bd4a{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:38682}
[junit4] 2> 2317770 INFO (closeThreadPool-7121-thread-2) [ ] o.e.j.s.Server Started @2317823ms
[junit4] 2> 2317770 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/tempDir-001/jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=38682, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-2-001/cores}
[junit4] 2> 2317770 ERROR (closeThreadPool-7121-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2317770 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 2317770 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 2317770 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2317770 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2317770 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-02-09T16:16:06.462Z
[junit4] 2> 2317773 INFO (zkConnectionManagerCallback-7140-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2317774 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2317774 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-2-001/solr.xml
[junit4] 2> 2317779 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2317779 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2317780 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2317784 INFO (zkConnectionManagerCallback-7138-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2317786 INFO (closeThreadPool-7121-thread-1) [n:127.0.0.1:37569_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2317787 INFO (closeThreadPool-7121-thread-1) [n:127.0.0.1:37569_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39082/solr ready
[junit4] 2> 2317787 INFO (closeThreadPool-7121-thread-1) [n:127.0.0.1:37569_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 2317815 INFO (closeThreadPool-7121-thread-1) [n:127.0.0.1:37569_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635
[junit4] 2> 2317830 INFO (closeThreadPool-7121-thread-1) [n:127.0.0.1:37569_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635
[junit4] 2> 2317831 INFO (closeThreadPool-7121-thread-1) [n:127.0.0.1:37569_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635
[junit4] 2> 2317833 INFO (closeThreadPool-7121-thread-1) [n:127.0.0.1:37569_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-1-001/cores
[junit4] 2> 2317925 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 2317927 WARN (closeThreadPool-7121-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@728e9615[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2317931 WARN (closeThreadPool-7121-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@6727a792[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2317933 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39082/solr
[junit4] 2> 2317936 INFO (zkConnectionManagerCallback-7147-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2317938 INFO (zkConnectionManagerCallback-7149-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2317947 INFO (closeThreadPool-7121-thread-2) [n:127.0.0.1:38682_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2317959 INFO (closeThreadPool-7121-thread-2) [n:127.0.0.1:38682_ ] o.a.s.c.ZkController Publish node=127.0.0.1:38682_ as DOWN
[junit4] 2> 2317969 INFO (closeThreadPool-7121-thread-2) [n:127.0.0.1:38682_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2317969 INFO (closeThreadPool-7121-thread-2) [n:127.0.0.1:38682_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38682_
[junit4] 2> 2317971 INFO (zkCallback-7119-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2317971 INFO (zkCallback-7100-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2317971 INFO (zkCallback-7107-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2317971 INFO (zkCallback-7130-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2317977 INFO (zkCallback-7148-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2317977 INFO (zkCallback-7137-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2317986 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-3-001 of type NRT
[junit4] 2> 2317986 WARN (closeThreadPool-7121-thread-1) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 2317986 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 2317986 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 2317986 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 2317988 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2317988 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2317988 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 2317988 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@734732cc{/,null,AVAILABLE}
[junit4] 2> 2317989 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@19dd742{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:35796}
[junit4] 2> 2317989 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.Server Started @2318041ms
[junit4] 2> 2317989 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/tempDir-001/jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=35796, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-3-001/cores}
[junit4] 2> 2317989 ERROR (closeThreadPool-7121-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2317989 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 2317989 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 2317989 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2317989 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2317989 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-02-09T16:16:06.681Z
[junit4] 2> 2317992 INFO (zkConnectionManagerCallback-7153-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2317995 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2317995 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-3-001/solr.xml
[junit4] 2> 2318000 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2318000 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2318035 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2318042 INFO (zkConnectionManagerCallback-7158-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2318044 INFO (closeThreadPool-7121-thread-2) [n:127.0.0.1:38682_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2318045 INFO (closeThreadPool-7121-thread-2) [n:127.0.0.1:38682_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39082/solr ready
[junit4] 2> 2318049 INFO (closeThreadPool-7121-thread-2) [n:127.0.0.1:38682_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 2318079 INFO (closeThreadPool-7121-thread-2) [n:127.0.0.1:38682_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635
[junit4] 2> 2318099 INFO (closeThreadPool-7121-thread-2) [n:127.0.0.1:38682_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635
[junit4] 2> 2318099 INFO (closeThreadPool-7121-thread-2) [n:127.0.0.1:38682_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635
[junit4] 2> 2318101 INFO (closeThreadPool-7121-thread-2) [n:127.0.0.1:38682_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-2-001/cores
[junit4] 2> 2318109 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 2318110 WARN (closeThreadPool-7121-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@4c285b09[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2318115 WARN (closeThreadPool-7121-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@3368aacd[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2318116 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39082/solr
[junit4] 2> 2318117 INFO (zkConnectionManagerCallback-7165-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2318122 INFO (zkConnectionManagerCallback-7167-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2318127 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2318131 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.ZkController Publish node=127.0.0.1:35796_ as DOWN
[junit4] 2> 2318132 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2318132 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35796_
[junit4] 2> 2318133 INFO (zkCallback-7137-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2318134 INFO (zkCallback-7148-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2318134 INFO (zkCallback-7119-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2318134 INFO (zkCallback-7130-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2318134 INFO (zkCallback-7107-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2318134 INFO (zkCallback-7100-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2318134 INFO (zkCallback-7157-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2318135 INFO (zkCallback-7166-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2318164 INFO (zkConnectionManagerCallback-7174-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2318166 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 2318167 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39082/solr ready
[junit4] 2> 2318168 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 2318199 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635
[junit4] 2> 2318215 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635
[junit4] 2> 2318215 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635
[junit4] 2> 2318217 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-3-001/cores
[junit4] 2> 2318234 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-4-001 of type NRT
[junit4] 2> 2318235 WARN (closeThreadPool-7121-thread-2) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 2318235 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 2318235 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 2318235 INFO (closeThreadPool-7121-thread-2) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 2318236 INFO (closeThreadPool-7121-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2318236 INFO (closeThreadPool-7121-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2318236 INFO (closeThreadPool-7121-thread-2) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 2318237 INFO (closeThreadPool-7121-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@701a6e38{/,null,AVAILABLE}
[junit4] 2> 2318237 INFO (closeThreadPool-7121-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@494d65be{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:35107}
[junit4] 2> 2318237 INFO (closeThreadPool-7121-thread-2) [ ] o.e.j.s.Server Started @2318290ms
[junit4] 2> 2318237 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/tempDir-001/jetty4, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=35107, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-4-001/cores}
[junit4] 2> 2318237 ERROR (closeThreadPool-7121-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2318237 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 2318238 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 2318238 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2318238 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2318238 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-02-09T16:16:06.930Z
[junit4] 2> 2318240 INFO (zkConnectionManagerCallback-7177-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2318240 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2318241 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-4-001/solr.xml
[junit4] 2> 2318246 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2318246 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2318269 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2318378 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 2318380 WARN (closeThreadPool-7121-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@2a4a833e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2318384 WARN (closeThreadPool-7121-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@2c6b3b3f[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2318385 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39082/solr
[junit4] 2> 2318387 INFO (zkConnectionManagerCallback-7183-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2318389 INFO (zkConnectionManagerCallback-7185-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2318395 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 2318399 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.ZkController Publish node=127.0.0.1:35107_ as DOWN
[junit4] 2> 2318400 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2318400 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35107_
[junit4] 2> 2318402 INFO (zkCallback-7137-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2318402 INFO (zkCallback-7100-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2318402 INFO (zkCallback-7166-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2318402 INFO (zkCallback-7148-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2318402 INFO (zkCallback-7107-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2318402 INFO (zkCallback-7157-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2318403 INFO (zkCallback-7173-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2318403 INFO (zkCallback-7184-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2318413 INFO (zkCallback-7130-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2318413 INFO (zkCallback-7119-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2318437 INFO (zkConnectionManagerCallback-7192-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2318438 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
[junit4] 2> 2318440 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39082/solr ready
[junit4] 2> 2318440 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 2318452 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[386ED1F1C26D0A10]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 5 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-5-001 of type NRT
[junit4] 2> 2318452 WARN (closeThreadPool-7121-thread-1) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 2318452 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 2318453 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 2318453 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 2318461 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2318461 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2318461 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 2318462 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@671e8c2f{/,null,AVAILABLE}
[junit4] 2> 2318463 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@62d2b5ac{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:46237}
[junit4] 2> 2318463 INFO (closeThreadPool-7121-thread-1) [ ] o.e.j.s.Server Started @2318516ms
[junit4] 2> 2318463 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/tempDir-001/jetty5, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=46237, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-5-001/cores}
[junit4] 2> 2318464 ERROR (closeThreadPool-7121-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2318464 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 2318464 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 2318464 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2318464 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2318464 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-02-09T16:16:07.156Z
[junit4] 2> 2318466 INFO (zkConnectionManagerCallback-7195-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2318467 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2318467 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_386ED1F1C26D0A10-001/shard-5-001/solr.xml
[junit4] 2> 2318471 INFO (closeThreadPool-7121-thread-2) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@40efc635
[junit4] 2> 2318472 INFO (closeThreadPool-7121-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFail
[...truncated too long message...]
= /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null1063198989
[copy] Copying 237 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null1063198989
[delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null1063198989
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 47ms :: artifacts dl 7ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 336 minutes 24 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2847)
at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2726)
at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2707)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3086)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1072)
at hudson.FilePath.act(FilePath.java:1061)
at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1835)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3088)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1074)
at hudson.FilePath.act(FilePath.java:1061)
at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1835)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
[JENKINS] Lucene-Solr-BadApples-NightlyTests-master - Build # 48 -
Still unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-master/48/
1 tests failed.
FAILED: org.apache.solr.cloud.RestartWhileUpdatingTest.test
Error Message:
There are still nodes recoverying - waited for 320 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
at __randomizedtesting.SeedInfo.seed([A8D95E0C3EC09BBD:208D61D6903CF645]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:195)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1038)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1595)
at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:144)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 14552 lines...]
[junit4] Suite: org.apache.solr.cloud.RestartWhileUpdatingTest
[junit4] 2> 2247456 INFO (SUITE-RestartWhileUpdatingTest-seed#[A8D95E0C3EC09BBD]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/init-core-data-001
[junit4] 2> 2247457 WARN (SUITE-RestartWhileUpdatingTest-seed#[A8D95E0C3EC09BBD]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 2247457 INFO (SUITE-RestartWhileUpdatingTest-seed#[A8D95E0C3EC09BBD]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 2247460 INFO (SUITE-RestartWhileUpdatingTest-seed#[A8D95E0C3EC09BBD]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 2247461 INFO (SUITE-RestartWhileUpdatingTest-seed#[A8D95E0C3EC09BBD]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 2247468 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2247468 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2247468 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2247568 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.ZkTestServer start zk server on port:42204
[junit4] 2> 2247568 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:42204
[junit4] 2> 2247568 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 42204
[junit4] 2> 2247576 INFO (zkConnectionManagerCallback-14834-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2247586 INFO (zkConnectionManagerCallback-14836-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2247587 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 2247589 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 2247590 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 2247591 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 2247593 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 2247594 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 2247595 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 2247596 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 2247598 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 2247599 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 2247600 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 2247603 INFO (zkConnectionManagerCallback-14840-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2247605 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 2247795 WARN (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 2247795 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 2247795 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 2247795 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 2247796 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2247796 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2247796 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 2247797 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5774b67e{/,null,AVAILABLE}
[junit4] 2> 2247797 WARN (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@68fcb677[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2247798 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@147d61aa{SSL,[ssl, http/1.1]}{127.0.0.1:33939}
[junit4] 2> 2247798 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.e.j.s.Server Started @2247874ms
[junit4] 2> 2247798 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/tempDir-001/control/data, hostContext=/, hostPort=33939, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/control-001/cores}
[junit4] 2> 2247798 ERROR (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2247798 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 2247799 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 2247799 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2247799 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2247799 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-02-02T14:46:15.694Z
[junit4] 2> 2247801 INFO (zkConnectionManagerCallback-14842-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2247802 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2247802 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/control-001/solr.xml
[junit4] 2> 2247807 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2247807 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2247809 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2247991 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 2247992 WARN (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 2247993 WARN (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@17c33166[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2247997 WARN (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 2247998 WARN (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@7884357e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2247999 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42204/solr
[junit4] 2> 2248000 INFO (zkConnectionManagerCallback-14848-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2248003 INFO (zkConnectionManagerCallback-14850-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2248100 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [n:127.0.0.1:33939_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33939_
[junit4] 2> 2248101 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [n:127.0.0.1:33939_ ] o.a.s.c.Overseer Overseer (id=72729569852915717-127.0.0.1:33939_-n_0000000000) starting
[junit4] 2> 2248110 INFO (zkConnectionManagerCallback-14857-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2248114 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [n:127.0.0.1:33939_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42204/solr ready
[junit4] 2> 2248125 INFO (OverseerStateUpdate-72729569852915717-127.0.0.1:33939_-n_0000000000) [n:127.0.0.1:33939_ ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:33939_
[junit4] 2> 2248126 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [n:127.0.0.1:33939_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33939_
[junit4] 2> 2248127 INFO (OverseerStateUpdate-72729569852915717-127.0.0.1:33939_-n_0000000000) [n:127.0.0.1:33939_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2248150 INFO (zkCallback-14856-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2248171 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [n:127.0.0.1:33939_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 2248222 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [n:127.0.0.1:33939_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c
[junit4] 2> 2248238 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [n:127.0.0.1:33939_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c
[junit4] 2> 2248238 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [n:127.0.0.1:33939_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c
[junit4] 2> 2248240 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [n:127.0.0.1:33939_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/control-001/cores
[junit4] 2> 2248274 INFO (zkConnectionManagerCallback-14863-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2248275 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2248277 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42204/solr ready
[junit4] 2> 2248298 INFO (qtp1780132406-47412) [n:127.0.0.1:33939_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:33939_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2248303 INFO (OverseerThreadFactory-6051-thread-1-processing-n:127.0.0.1:33939_) [n:127.0.0.1:33939_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 2248415 INFO (qtp1780132406-47410) [n:127.0.0.1:33939_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2248417 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2248418 INFO (qtp1780132406-47408) [n:127.0.0.1:33939_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2248419 INFO (qtp1780132406-47409) [n:127.0.0.1:33939_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2248421 INFO (qtp1780132406-47410) [n:127.0.0.1:33939_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2248446 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2248447 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2249480 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 2249497 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 2249618 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 2249640 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 2249641 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c
[junit4] 2> 2249641 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 2249646 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=22, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.583984375, noCFSRatio=0.6344138601555405]
[junit4] 2> 2249659 WARN (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2249855 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2249855 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2249858 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 30000ms;
[junit4] 2> 2249858 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
[junit4] 2> 2249860 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5275168808969725]
[junit4] 2> 2249866 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@37842ec3[control_collection_shard1_replica_n1] main]
[junit4] 2> 2249874 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2249874 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2249875 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2249875 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1624368771518955520
[junit4] 2> 2249879 INFO (searcherExecutor-6056-thread-1-processing-n:127.0.0.1:33939_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@37842ec3[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2249893 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 2249893 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
[junit4] 2> 2249899 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2249899 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2249899 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:33939/control_collection_shard1_replica_n1/
[junit4] 2> 2249900 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2249900 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:33939/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 2249900 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72729569852915717-core_node2-n_0000000000
[junit4] 2> 2249902 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:33939/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 2250004 INFO (zkCallback-14849-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2250005 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2250007 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1561
[junit4] 2> 2250030 INFO (zkCallback-14849-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2250033 INFO (qtp1780132406-47412) [n:127.0.0.1:33939_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 2250132 INFO (zkCallback-14849-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2250132 INFO (zkCallback-14849-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2250133 INFO (zkCallback-14849-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2250134 INFO (qtp1780132406-47412) [n:127.0.0.1:33939_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:33939_&wt=javabin&version=2} status=0 QTime=1836
[junit4] 2> 2250143 INFO (zkConnectionManagerCallback-14869-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2250144 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2250146 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42204/solr ready
[junit4] 2> 2250146 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 2250150 INFO (qtp1780132406-47408) [n:127.0.0.1:33939_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2250153 INFO (OverseerThreadFactory-6051-thread-2-processing-n:127.0.0.1:33939_) [n:127.0.0.1:33939_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 2250154 INFO (OverseerCollectionConfigSetProcessor-72729569852915717-127.0.0.1:33939_-n_0000000000) [n:127.0.0.1:33939_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2250359 WARN (OverseerThreadFactory-6051-thread-2-processing-n:127.0.0.1:33939_) [n:127.0.0.1:33939_ ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 2250363 INFO (qtp1780132406-47408) [n:127.0.0.1:33939_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 2250364 INFO (qtp1780132406-47408) [n:127.0.0.1:33939_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=213
[junit4] 2> 2250365 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=3
[junit4] 2> 2250548 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/shard-1-001 of type NRT
[junit4] 2> 2250549 WARN (closeThreadPool-14870-thread-1) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 2250550 INFO (closeThreadPool-14870-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 2250550 INFO (closeThreadPool-14870-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 2250550 INFO (closeThreadPool-14870-thread-1) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 2250551 INFO (closeThreadPool-14870-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2250551 INFO (closeThreadPool-14870-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2250551 INFO (closeThreadPool-14870-thread-1) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 2250552 INFO (closeThreadPool-14870-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@52a6d29a{/,null,AVAILABLE}
[junit4] 2> 2250552 WARN (closeThreadPool-14870-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@72728ad[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2250553 INFO (closeThreadPool-14870-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2527854f{SSL,[ssl, http/1.1]}{127.0.0.1:35846}
[junit4] 2> 2250553 INFO (closeThreadPool-14870-thread-1) [ ] o.e.j.s.Server Started @2250629ms
[junit4] 2> 2250553 INFO (closeThreadPool-14870-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=35846, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/shard-1-001/cores}
[junit4] 2> 2250553 ERROR (closeThreadPool-14870-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2250553 INFO (closeThreadPool-14870-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 2250553 INFO (closeThreadPool-14870-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 2250553 INFO (closeThreadPool-14870-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2250553 INFO (closeThreadPool-14870-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2250553 INFO (closeThreadPool-14870-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-02-02T14:46:18.448Z
[junit4] 2> 2250555 INFO (zkConnectionManagerCallback-14872-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2250556 INFO (closeThreadPool-14870-thread-1) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2250557 INFO (closeThreadPool-14870-thread-1) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/shard-1-001/solr.xml
[junit4] 2> 2250562 INFO (closeThreadPool-14870-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2250562 INFO (closeThreadPool-14870-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2250564 INFO (closeThreadPool-14870-thread-1) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2250754 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/shard-2-001 of type NRT
[junit4] 2> 2250758 WARN (closeThreadPool-14870-thread-2) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 2250759 INFO (closeThreadPool-14870-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 2250759 INFO (closeThreadPool-14870-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 2250759 INFO (closeThreadPool-14870-thread-2) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 2250760 INFO (closeThreadPool-14870-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2250760 INFO (closeThreadPool-14870-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2250760 INFO (closeThreadPool-14870-thread-2) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 2250760 INFO (closeThreadPool-14870-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4f1c1850{/,null,AVAILABLE}
[junit4] 2> 2250761 WARN (closeThreadPool-14870-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@196ef1e3[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2250761 INFO (closeThreadPool-14870-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6252181e{SSL,[ssl, http/1.1]}{127.0.0.1:45995}
[junit4] 2> 2250762 INFO (closeThreadPool-14870-thread-2) [ ] o.e.j.s.Server Started @2250837ms
[junit4] 2> 2250762 INFO (closeThreadPool-14870-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=45995, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/shard-2-001/cores}
[junit4] 2> 2250762 ERROR (closeThreadPool-14870-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2250762 INFO (closeThreadPool-14870-thread-2) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 2250762 INFO (closeThreadPool-14870-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 2250762 INFO (closeThreadPool-14870-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2250762 INFO (closeThreadPool-14870-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2250762 INFO (closeThreadPool-14870-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-02-02T14:46:18.657Z
[junit4] 2> 2250765 INFO (zkConnectionManagerCallback-14874-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2250766 INFO (closeThreadPool-14870-thread-2) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2250766 INFO (closeThreadPool-14870-thread-2) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/shard-2-001/solr.xml
[junit4] 2> 2250771 INFO (closeThreadPool-14870-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2250771 INFO (closeThreadPool-14870-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2250773 INFO (closeThreadPool-14870-thread-2) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2250880 INFO (closeThreadPool-14870-thread-2) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 2250881 WARN (closeThreadPool-14870-thread-2) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 2250886 WARN (closeThreadPool-14870-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@7d50f70e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2250890 WARN (closeThreadPool-14870-thread-2) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 2250890 WARN (closeThreadPool-14870-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@15f42009[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2250892 INFO (closeThreadPool-14870-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42204/solr
[junit4] 2> 2250917 INFO (zkConnectionManagerCallback-14880-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2250920 INFO (closeThreadPool-14870-thread-1) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 2250921 WARN (closeThreadPool-14870-thread-1) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 2250923 WARN (closeThreadPool-14870-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@5ffd15a0[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2250926 INFO (zkConnectionManagerCallback-14882-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2250926 WARN (closeThreadPool-14870-thread-1) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 2250929 WARN (closeThreadPool-14870-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@4c0bd1a3[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2250930 INFO (closeThreadPool-14870-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42204/solr
[junit4] 2> 2250934 INFO (zkConnectionManagerCallback-14890-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2250939 INFO (closeThreadPool-14870-thread-2) [n:127.0.0.1:45995_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2250941 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x10263288fc0000d, likely client has closed socket
[junit4] 2> 2250942 INFO (zkConnectionManagerCallback-14892-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2250945 INFO (closeThreadPool-14870-thread-2) [n:127.0.0.1:45995_ ] o.a.s.c.ZkController Publish node=127.0.0.1:45995_ as DOWN
[junit4] 2> 2250947 INFO (closeThreadPool-14870-thread-2) [n:127.0.0.1:45995_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2250947 INFO (closeThreadPool-14870-thread-2) [n:127.0.0.1:45995_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45995_
[junit4] 2> 2250948 INFO (zkCallback-14849-thread-4) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2250952 INFO (zkCallback-14856-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2250952 INFO (zkCallback-14868-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2250954 INFO (zkCallback-14881-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2250954 INFO (closeThreadPool-14870-thread-1) [n:127.0.0.1:35846_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2250958 INFO (closeThreadPool-14870-thread-1) [n:127.0.0.1:35846_ ] o.a.s.c.ZkController Publish node=127.0.0.1:35846_ as DOWN
[junit4] 2> 2250959 INFO (closeThreadPool-14870-thread-1) [n:127.0.0.1:35846_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2250959 INFO (closeThreadPool-14870-thread-1) [n:127.0.0.1:35846_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35846_
[junit4] 2> 2250961 INFO (zkCallback-14856-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2250961 INFO (zkCallback-14849-thread-4) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2250961 INFO (zkCallback-14868-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2250962 INFO (zkCallback-14891-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2250962 INFO (zkCallback-14881-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2250989 INFO (zkConnectionManagerCallback-14899-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2250990 INFO (zkConnectionManagerCallback-14904-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2250991 INFO (closeThreadPool-14870-thread-2) [n:127.0.0.1:45995_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2250991 INFO (closeThreadPool-14870-thread-1) [n:127.0.0.1:35846_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2250992 INFO (closeThreadPool-14870-thread-1) [n:127.0.0.1:35846_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42204/solr ready
[junit4] 2> 2250992 INFO (closeThreadPool-14870-thread-2) [n:127.0.0.1:45995_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42204/solr ready
[junit4] 2> 2250993 INFO (closeThreadPool-14870-thread-2) [n:127.0.0.1:45995_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 2250998 INFO (closeThreadPool-14870-thread-1) [n:127.0.0.1:35846_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 2251005 WARN (closeThreadPool-14870-thread-2) [n:127.0.0.1:45995_ ] o.a.s.m.r.j.JmxMetricsReporter Unable to register meter
[junit4] 2> javax.management.InstanceNotFoundException: solr:dom1=node,category=QUERY,scope=/admin/metrics/history,name=timeouts
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1095) ~[?:1.8.0_191]
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:427) ~[?:1.8.0_191]
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415) ~[?:1.8.0_191]
[junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546) ~[?:1.8.0_191]
[junit4] 2> at org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.registerMBean(JmxMetricsReporter.java:531) ~[java/:?]
[junit4] 2> at org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.onMeterAdded(JmxMetricsReporter.java:648) ~[java/:?]
[junit4] 2> at com.codahale.metrics.MetricRegistry.notifyListenerOfAddedMetric(MetricRegistry.java:460) ~[metrics-core-3.2.6.jar:3.2.6]
[junit4] 2> at com.codahale.metrics.MetricRegistry.addListener(MetricRegistry.java:297) ~[metrics-core-3.2.6.jar:3.2.6]
[junit4] 2> at org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.start(JmxMetricsReporter.java:729) ~[java/:?]
[junit4] 2> at org.apache.solr.metrics.reporters.SolrJmxReporter.doInit(SolrJmxReporter.java:110) ~[java/:?]
[junit4] 2> at org.apache.solr.metrics.SolrMetricReporter.init(SolrMetricReporter.java:70) ~[java/:?]
[junit4] 2> at org.apache.solr.metrics.SolrMetricManager.loadReporter(SolrMetricManager.java:916) ~[java/:?]
[junit4] 2> at org.apache.solr.metrics.SolrMetricManager.loadReporters(SolrMetricManager.java:843) ~[java/:?]
[junit4] 2> at org.apache.solr.core.CoreContainer.load(CoreContainer.java:600) ~[java/:?]
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:252) ~[java/:?]
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:172) ~[java/:?]
[junit4] 2> at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:136) ~[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.lambda$initialize$0(ServletHandler.java:750) ~[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948) ~[?:1.8.0_191]
[junit4] 2> at java.util.stream.Streams$ConcatSpliterator.forEachRemaining(Streams.java:742) ~[?:1.8.0_191]
[junit4] 2> at java.util.stream.Streams$ConcatSpliterator.forEachRemaining(Streams.java:742) ~[?:1.8.0_191]
[junit4] 2> at java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:580) ~[?:1.8.0_191]
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:744) ~[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1449) ~[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1513) ~[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1158) ~[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:995) ~[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:467) ~[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:381) ~[java/:?]
[junit4] 2> at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) ~[jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) ~[jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:544) ~[java/:?]
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:483) ~[java/:?]
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:451) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.AbstractFullDistribZkTestBase.lambda$createJettys$2(AbstractFullDistribZkTestBase.java:464) ~[java/:?]
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_191]
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_191]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_191]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_191]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
[junit4] 2> 2251012 INFO (TEST-RestartWhileUpdatingTest.test-seed#[A8D95E0C3EC09BBD]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/shard-3-001 of type NRT
[junit4] 2> 2251043 WARN (closeThreadPool-14870-thread-3) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 2251044 INFO (closeThreadPool-14870-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 2251044 INFO (closeThreadPool-14870-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 2251044 INFO (closeThreadPool-14870-thread-3) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 2251045 INFO (closeThreadPool-14870-thread-3) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2251045 INFO (closeThreadPool-14870-thread-3) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2251045 INFO (closeThreadPool-14870-thread-3) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 2251046 INFO (closeThreadPool-14870-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6f073f86{/,null,AVAILABLE}
[junit4] 2> 2251046 WARN (closeThreadPool-14870-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@2f10c7fc[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2251047 INFO (closeThreadPool-14870-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5ca0b56a{SSL,[ssl, http/1.1]}{127.0.0.1:46045}
[junit4] 2> 2251047 INFO (closeThreadPool-14870-thread-3) [ ] o.e.j.s.Server Started @2251123ms
[junit4] 2> 2251047 INFO (closeThreadPool-14870-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=46045, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/shard-3-001/cores}
[junit4] 2> 2251047 ERROR (closeThreadPool-14870-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2251047 INFO (closeThreadPool-14870-thread-3) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 2251047 INFO (closeThreadPool-14870-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 2251047 INFO (closeThreadPool-14870-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2251047 INFO (closeThreadPool-14870-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2251047 INFO (closeThreadPool-14870-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-02-02T14:46:18.942Z
[junit4] 2> 2251049 INFO (zkConnectionManagerCallback-14908-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2251050 INFO (closeThreadPool-14870-thread-3) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2251050 INFO (closeThreadPool-14870-thread-3) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/shard-3-001/solr.xml
[junit4] 2> 2251055 INFO (closeThreadPool-14870-thread-1) [n:127.0.0.1:35846_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c
[junit4] 2> 2251056 INFO (closeThreadPool-14870-thread-3) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2251056 INFO (closeThreadPool-14870-thread-3) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2251058 INFO (closeThreadPool-14870-thread-3) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2251060 INFO (closeThreadPool-14870-thread-2) [n:127.0.0.1:45995_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c
[junit4] 2> 2251076 INFO (closeThreadPool-14870-thread-1) [n:127.0.0.1:35846_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c
[junit4] 2> 2251076 INFO (closeThreadPool-14870-thread-1) [n:127.0.0.1:35846_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c
[junit4] 2> 2251078 INFO (closeThreadPool-14870-thread-1) [n:127.0.0.1:35846_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/shard-1-001/cores
[junit4] 2> 2251079 INFO (closeThreadPool-14870-thread-2) [n:127.0.0.1:45995_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c
[junit4] 2> 2251079 INFO (closeThreadPool-14870-thread-2) [n:127.0.0.1:45995_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c
[junit4] 2> 2251081 INFO (closeThreadPool-14870-thread-2) [n:127.0.0.1:45995_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/shard-2-001/cores
[junit4] 2> 2251141 INFO (closeThreadPool-14870-thread-3) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 2251142 WARN (closeThreadPool-14870-thread-3) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 2251142 WARN (closeThreadPool-14870-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@6304395e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2251146 WARN (closeThreadPool-14870-thread-3) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 2251147 WARN (closeThreadPool-14870-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@6c8b7137[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 2251148 INFO (closeThreadPool-14870-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42204/solr
[junit4] 2> 2251149 INFO (zkConnectionManagerCallback-14914-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2251152 INFO (zkConnectionManagerCallback-14916-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2251157 INFO (closeThreadPool-14870-thread-3) [n:127.0.0.1:46045_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2251162 INFO (closeThreadPool-14870-thread-3) [n:127.0.0.1:46045_ ] o.a.s.c.ZkController Publish node=127.0.0.1:46045_ as DOWN
[junit4] 2> 2251163 INFO (closeThreadPool-14870-thread-3) [n:127.0.0.1:46045_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2251163 INFO (closeThreadPool-14870-thread-3) [n:127.0.0.1:46045_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46045_
[junit4] 2> 2251165 INFO (zkCallback-14849-thread-4) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2251165 INFO (zkCallback-14868-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2251166 INFO (zkCallback-14881-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2251166 INFO (zkCallback-14856-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2251171 INFO (zkCallback-14891-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2251177 INFO (zkCallback-14903-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2251186 INFO (zkCallback-14915-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2251187 INFO (zkCallback-14898-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2251209 INFO (zkConnectionManagerCallback-14923-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2251210 INFO (closeThreadPool-14870-thread-3) [n:127.0.0.1:46045_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 2251212 INFO (closeThreadPool-14870-thread-3) [n:127.0.0.1:46045_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42204/solr ready
[junit4] 2> 2251212 INFO (closeThreadPool-14870-thread-3) [n:127.0.0.1:46045_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 2251244 INFO (closeThreadPool-14870-thread-3) [n:127.0.0.1:46045_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c
[junit4] 2> 2251263 INFO (closeThreadPool-14870-thread-3) [n:127.0.0.1:46045_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c
[junit4] 2> 2251264 INFO (closeThreadPool-14870-thread-3) [n:127.0.0.1:46045_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c
[junit4] 2> 2251266 INFO (closeThreadPool-14870-thread-3) [n:127.0.0.1:46045_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/shard-3-001/cores
[junit4] 2> 2251349 INFO (qtp1780132406-47409) [n:127.0.0.1:33939_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:35846_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2251351 INFO (qtp1780132406-47411) [n:127.0.0.1:33939_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:46045_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2251351 INFO (qtp1780132406-47410) [n:127.0.0.1:33939_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:45995_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2251356 INFO (OverseerCollectionConfigSetProcessor-72729569852915717-127.0.0.1:33939_-n_0000000000) [n:127.0.0.1:33939_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2251364 INFO (qtp1780132406-47412) [n:127.0.0.1:33939_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=2
[junit4] 2> 2251365 INFO (qtp1780132406-47408) [n:127.0.0.1:33939_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 2251366 INFO (qtp1780132406-47412) [n:127.0.0.1:33939_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 2251367 INFO (qtp1780132406-47408) [n:127.0.0.1:33939_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251368 INFO (qtp1780132406-47412) [n:127.0.0.1:33939_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251371 INFO (qtp1780132406-47408) [n:127.0.0.1:33939_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251373 INFO (qtp1780132406-47412) [n:127.0.0.1:33939_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251374 INFO (qtp1780132406-47408) [n:127.0.0.1:33939_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251395 INFO (qtp1243170961-47475) [n:127.0.0.1:35846_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251397 INFO (qtp1243170961-47471) [n:127.0.0.1:35846_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251398 INFO (qtp1243170961-47472) [n:127.0.0.1:35846_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251399 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251400 INFO (qtp1243170961-47474) [n:127.0.0.1:35846_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251420 INFO (qtp289687245-47484) [n:127.0.0.1:45995_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251421 INFO (qtp289687245-47485) [n:127.0.0.1:45995_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251424 INFO (qtp289687245-47486) [n:127.0.0.1:45995_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251425 INFO (qtp289687245-47487) [n:127.0.0.1:45995_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251426 INFO (qtp289687245-47488) [n:127.0.0.1:45995_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251443 INFO (qtp473342491-47538) [n:127.0.0.1:46045_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251444 INFO (qtp473342491-47534) [n:127.0.0.1:46045_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251445 INFO (qtp473342491-47535) [n:127.0.0.1:46045_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251446 INFO (qtp473342491-47536) [n:127.0.0.1:46045_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251447 INFO (qtp473342491-47537) [n:127.0.0.1:46045_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2251448 INFO (OverseerThreadFactory-6051-thread-3-processing-n:127.0.0.1:33939_) [n:127.0.0.1:33939_ c:collection1 s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:35846_ for creating new replica of shard shard1 for collection collection1
[junit4] 2> 2251451 INFO (OverseerThreadFactory-6051-thread-3-processing-n:127.0.0.1:33939_) [n:127.0.0.1:33939_ c:collection1 s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 2251480 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ x:collection1_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n1&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2252516 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 2252578 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema [collection1_shard1_replica_n1] Schema name=test
[junit4] 2> 2252677 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 2252712 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n1' using configuration from collection collection1, trusted=true
[junit4] 2> 2252713 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n1' (registry 'solr.core.collection1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ea6e21c
[junit4] 2> 2252713 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [[collection1_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/shard-1-001/cores/collection1_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.RestartWhileUpdatingTest_A8D95E0C3EC09BBD-001/shard-1-001/cores/collection1_shard1_replica_n1/data/]
[junit4] 2> 2252717 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=22, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.583984375, noCFSRatio=0.6344138601555405]
[junit4] 2> 2252732 WARN (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2252819 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2252819 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2252822 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 30000ms;
[junit4] 2> 2252822 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
[junit4] 2> 2252823 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5275168808969725]
[junit4] 2> 2252824 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1da7af9c[collection1_shard1_replica_n1] main]
[junit4] 2> 2252826 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2252826 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2252827 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2252827 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1624368774614351872
[junit4] 2> 2252831 INFO (searcherExecutor-6088-thread-1-processing-n:127.0.0.1:35846_ x:collection1_shard1_replica_n1 c:collection1 s:shard1) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [collection1_shard1_replica_n1] Registered new searcher Searcher@1da7af9c[collection1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2252834 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 2252834 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/collection1/leaders/shard1
[junit4] 2> 2252838 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2252838 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2252838 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:35846/collection1_shard1_replica_n1/
[junit4] 2> 2252838 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2252839 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:35846/collection1_shard1_replica_n1/ has no replicas
[junit4] 2> 2252839 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/collection1/leaders/shard1/leader after winning as /collections/collection1/leader_elect/shard1/election/72729569852915726-core_node2-n_0000000000
[junit4] 2> 2252841 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:35846/collection1_shard1_replica_n1/ shard1
[junit4] 2> 2252943 INFO (zkCallback-14891-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2252943 INFO (zkCallback-14891-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2252945 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2252949 INFO (qtp1243170961-47473) [n:127.0.0.1:35846_ c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n1&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1469
[junit4] 2> 2252952 INFO (qtp1780132406-47409) [n:127.0.0.1:33939_ c:collection1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:35846_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1603
[junit4] 2> 2253047 INFO (zkCallback-14891-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2253047 INFO (zkCallback-14891-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2253048 INFO (zkCallback-14891-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2253366 INFO (OverseerCollectionConfigSetProcessor-72729569852915717-127.0.0.1:33939_-n_0000000000) [n:127.0.0.1:33939_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[...truncated too long message...]
le = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null487544769
[copy] Copying 237 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null487544769
[delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null487544769
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 31ms :: artifacts dl 10ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 314 minutes 41 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2847)
at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2726)
at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2707)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3086)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1072)
at hudson.FilePath.act(FilePath.java:1061)
at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1835)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3088)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1074)
at hudson.FilePath.act(FilePath.java:1061)
at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1835)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
[JENKINS] Lucene-Solr-BadApples-NightlyTests-master - Build # 47 -
Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-master/47/
11 tests failed.
FAILED: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test
Error Message:
Test abandoned because suite timeout was reached.
Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
at __randomizedtesting.SeedInfo.seed([31196B5ADAAA2ED6]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([31196B5ADAAA2ED6]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
Error Message:
Captured an uncaught exception in thread: Thread[id=127710, name=Thread-11980, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=127710, name=Thread-11980, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:33776//control_collection
at __randomizedtesting.SeedInfo.seed([31196B5ADAAA2ED6]:0)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:638)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:33776//control_collection
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:661)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:256)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:245)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:224)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:636)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:549)
... 5 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
Error Message:
Captured an uncaught exception in thread: Thread[id=127711, name=Thread-11981, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=127711, name=Thread-11981, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:33776/collection2_shard7_replica_n36
at __randomizedtesting.SeedInfo.seed([31196B5ADAAA2ED6]:0)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:638)
Caused by: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:33776/collection2_shard7_replica_n36
at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:567)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1019)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:224)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:635)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:33776/collection2_shard7_replica_n36
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:661)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:256)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:245)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:213)
at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:561)
... 6 more
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:549)
... 12 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
Error Message:
Captured an uncaught exception in thread: Thread[id=127044, name=OverseerStateUpdate-74080050190811140-127.0.0.1:33776_-n_0000000000, state=RUNNABLE, group=Overseer state updater.]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=127044, name=OverseerStateUpdate-74080050190811140-127.0.0.1:33776_-n_0000000000, state=RUNNABLE, group=Overseer state updater.]
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
at __randomizedtesting.SeedInfo.seed([31196B5ADAAA2ED6]:0)
at org.apache.zookeeper.ZooKeeper.prependChroot(ZooKeeper.java:709)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1211)
at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:358)
at org.apache.solr.common.cloud.SolrZkClient$$Lambda$36/1463209208.execute(Unknown Source)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:71)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:358)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:432)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:150)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
Error Message:
Captured an uncaught exception in thread: Thread[id=127709, name=Thread-11979, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=127709, name=Thread-11979, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:33776//control_collection: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 500 Server Error</title>
</head>
<body><h2>HTTP ERROR 500</h2>
<p>Problem accessing /control_collection/update. Reason:
<pre> Server Error</pre></p><h3>Caused by:</h3><pre>java.lang.OutOfMemoryError: GC overhead limit exceeded
at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectField.<init>(DirectPostingsFormat.java:452)
at org.apache.lucene.codecs.memory.DirectPostingsFormat$DirectFields.<init>(DirectPostingsFormat.java:131)
at org.apache.lucene.codecs.memory.DirectPostingsFormat.fieldsProducer(DirectPostingsFormat.java:115)
at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:288)
at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:368)
at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:113)
at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:83)
at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:172)
at org.apache.lucene.index.BufferedUpdatesStream$SegmentState.<init>(BufferedUpdatesStream.java:271)
at org.apache.lucene.index.FrozenBufferedUpdates.openSegmentStates(FrozenBufferedUpdates.java:329)
at org.apache.lucene.index.FrozenBufferedUpdates.forceApply(FrozenBufferedUpdates.java:222)
at org.apache.lucene.index.FrozenBufferedUpdates.tryApply(FrozenBufferedUpdates.java:160)
at org.apache.lucene.index.IndexWriter.lambda$publishFrozenUpdates$3(IndexWriter.java:2589)
at org.apache.lucene.index.IndexWriter$$Lambda$466/43019974.process(Unknown Source)
at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5112)
at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1597)
at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1586)
at org.apache.solr.update.DirectUpdateHandler2.updateDocOrDocValues(DirectUpdateHandler2.java:970)
at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:342)
at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:289)
at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:236)
at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:76)
at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)
at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:995)
at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1216)
at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:700)
at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103)
at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:327)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readIterator(JavaBinUpdateRequestCodec.java:280)
at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:333)
at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:278)
</pre>
<hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.14.v20181114</a><hr/>
</body>
</html>
at __randomizedtesting.SeedInfo.seed([31196B5ADAAA2ED6]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:614)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:256)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:245)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:224)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:636)
FAILED: org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv_stored
Error Message:
Some docs had errors -- check logs expected:<0> but was:<3>
Stack Trace:
java.lang.AssertionError: Some docs had errors -- check logs expected:<0> but was:<3>
at __randomizedtesting.SeedInfo.seed([31196B5ADAAA2ED6:A3250ADE66ED130E]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.failNotEquals(Assert.java:834)
at org.junit.Assert.assertEquals(Assert.java:645)
at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.checkField(TestStressCloudBlindAtomicUpdates.java:345)
at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv_stored(TestStressCloudBlindAtomicUpdates.java:212)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TriLevelCompositeIdRoutingTest.test
Error Message:
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /solr/collections/collection1/terms/shard2
Stack Trace:
java.lang.RuntimeException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /solr/collections/collection1/terms/shard2
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForActiveReplicaCount(AbstractFullDistribZkTestBase.java:601)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:552)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:358)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1080)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /solr/collections/collection1/terms/shard2
at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1221)
at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:358)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:71)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:358)
at org.apache.solr.common.cloud.SolrZkClient.printLayout(SolrZkClient.java:615)
at org.apache.solr.common.cloud.SolrZkClient.printLayout(SolrZkClient.java:640)
at org.apache.solr.common.cloud.SolrZkClient.printLayout(SolrZkClient.java:640)
at org.apache.solr.common.cloud.SolrZkClient.printLayout(SolrZkClient.java:640)
at org.apache.solr.common.cloud.SolrZkClient.printLayout(SolrZkClient.java:640)
at org.apache.solr.common.cloud.SolrZkClient.printLayout(SolrZkClient.java:640)
at org.apache.solr.common.cloud.SolrZkClient.printLayoutToStream(SolrZkClient.java:660)
at org.apache.solr.common.cloud.SolrZkClient.printLayoutToStdOut(SolrZkClient.java:655)
at org.apache.solr.cloud.AbstractDistribZkTestBase.printLayout(AbstractDistribZkTestBase.java:315)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForActiveReplicaCount(AbstractFullDistribZkTestBase.java:599)
... 35 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TriLevelCompositeIdRoutingTest
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([31196B5ADAAA2ED6]:0)
FAILED: org.apache.solr.core.TestJmxIntegration.testJmxOnCoreReload
Error Message:
Number of registered MBeans is not the same as the number of core metrics: 443 != 442
Stack Trace:
java.lang.AssertionError: Number of registered MBeans is not the same as the number of core metrics: 443 != 442
at __randomizedtesting.SeedInfo.seed([31196B5ADAAA2ED6:33A0B036A9E9F21B]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.apache.solr.core.TestJmxIntegration.testJmxOnCoreReload(TestJmxIntegration.java:260)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.metrics.reporters.SolrJmxReporterCloudTest
Error Message:
Clean up static fields (in @AfterClass?), your test seems to hang on to approximately 11,572,176 bytes (threshold is 10,485,760). Field reference sizes (counted individually): - 11,571,456 bytes, private static javax.management.MBeanServer org.apache.solr.metrics.reporters.SolrJmxReporterCloudTest.mBeanServer - 192 bytes, public static org.junit.rules.TestRule org.apache.solr.SolrTestCaseJ4.solrClassRules - 128 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.factoryProp - 112 bytes, private static java.lang.String org.apache.solr.metrics.reporters.SolrJmxReporterCloudTest.COLLECTION - 96 bytes, private static java.util.Map org.apache.solr.SolrTestCaseJ4.savedClassLogLevels - 80 bytes, protected static volatile java.util.concurrent.ExecutorService org.apache.solr.SolrTestCaseJ4.testExecutor - 64 bytes, protected static java.lang.String org.apache.solr.SolrTestCaseJ4.coreName - 48 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.initialRootLogLevel
Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?), your test seems to hang on to approximately 11,572,176 bytes (threshold is 10,485,760). Field reference sizes (counted individually):
- 11,571,456 bytes, private static javax.management.MBeanServer org.apache.solr.metrics.reporters.SolrJmxReporterCloudTest.mBeanServer
- 192 bytes, public static org.junit.rules.TestRule org.apache.solr.SolrTestCaseJ4.solrClassRules
- 128 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.factoryProp
- 112 bytes, private static java.lang.String org.apache.solr.metrics.reporters.SolrJmxReporterCloudTest.COLLECTION
- 96 bytes, private static java.util.Map org.apache.solr.SolrTestCaseJ4.savedClassLogLevels
- 80 bytes, protected static volatile java.util.concurrent.ExecutorService org.apache.solr.SolrTestCaseJ4.testExecutor
- 64 bytes, protected static java.lang.String org.apache.solr.SolrTestCaseJ4.coreName
- 48 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.initialRootLogLevel
at __randomizedtesting.SeedInfo.seed([31196B5ADAAA2ED6]:0)
at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:170)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13889 lines...]
[junit4] Suite: org.apache.solr.core.TestJmxIntegration
[junit4] 2> 1301396 INFO (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.core.TestJmxIntegration_31196B5ADAAA2ED6-001/init-core-data-001
[junit4] 2> 1301397 WARN (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=6 numCloses=6
[junit4] 2> 1301397 INFO (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 1301399 INFO (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 1301400 INFO (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore
[junit4] 2> 1301401 INFO (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 1301435 INFO (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 1301478 INFO (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.s.IndexSchema [null] Schema name=test
[junit4] 2> 1301673 INFO (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1301950 INFO (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 1301951 WARN (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 1301958 WARN (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@3a212e77[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1302030 WARN (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 1302061 WARN (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@1fc8b9e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1302106 INFO (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1302106 INFO (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1302142 INFO (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@617d483
[junit4] 2> 1302148 INFO (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@617d483
[junit4] 2> 1302148 INFO (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@617d483
[junit4] 2> 1302150 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 1302189 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 1302221 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1302415 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1302423 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1, trusted=true
[junit4] 2> 1302423 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@617d483
[junit4] 2> 1302424 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.core.TestJmxIntegration_31196B5ADAAA2ED6-001/init-core-data-001/]
[junit4] 2> 1302436 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=35, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=22.958984375, floorSegmentMB=1.3056640625, forceMergeDeletesPctAllowed=10.007169334770646, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.30752591381726735, deletesPctAllowed=32.41259637393401
[junit4] 2> 1302509 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1302509 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1302511 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1302512 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1302526 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=49, maxMergedSegmentMB=91.642578125, floorSegmentMB=0.4658203125, forceMergeDeletesPctAllowed=21.025216847150237, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=26.783711149399203
[junit4] 2> 1302526 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6cb9ab20[collection1] main]
[junit4] 2> 1302527 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 1302536 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 1302635 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1302635 INFO (coreLoadExecutor-2557-thread-1) [ x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1623739922262261760
[junit4] 2> 1302651 INFO (searcherExecutor-2558-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 1302654 INFO (SUITE-TestJmxIntegration-seed#[31196B5ADAAA2ED6]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore end
[junit4] 2> 1302656 INFO (searcherExecutor-2558-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 1302656 INFO (searcherExecutor-2558-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 1302656 INFO (searcherExecutor-2558-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 1302656 INFO (searcherExecutor-2558-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 1302657 INFO (searcherExecutor-2558-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 1302657 INFO (searcherExecutor-2558-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 1302658 INFO (searcherExecutor-2558-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 1302658 INFO (searcherExecutor-2558-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 1302666 INFO (searcherExecutor-2558-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6cb9ab20[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1302677 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testJmxOnCoreReload
[junit4] 2> 1302677 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ ] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
[junit4] 2> 1302678 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@6e96300f[collection1] realtime]
[junit4] 2> 1302679 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:* (-9223372036854775807)} 0 1
[junit4] 2> 1302679 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1623739922308399104,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1302679 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@2e0133a5 commitCommandVersion:1623739922308399104
[junit4] 2> 1302689 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@2f4475ab[collection1] main]
[junit4] 2> 1302691 INFO (searcherExecutor-2558-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2f4475ab[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1302691 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1302691 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{commit=} 0 12
[junit4] 2> 1302694 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ ] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=*:*&wt=xml} hits=0 status=0 QTime=0
[junit4] 2> 1302815 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ ] o.a.s.c.TestJmxIntegration Before Reload: size of all core metrics: 443 MBeans: 443
[junit4] 2> 1302815 ERROR (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ ] o.a.s.c.CorePropertiesLocator Couldn't load core descriptor from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/core.properties:java.nio.file.NoSuchFileException: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/core.properties
[junit4] 2> 1302815 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 1302880 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 1302917 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ ] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1303162 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1303187 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ ] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1
[junit4] 2> 1303287 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@617d483
[junit4] 2> 1303288 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.core.TestJmxIntegration_31196B5ADAAA2ED6-001/init-core-data-001/]
[junit4] 2> 1303424 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1303424 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1303426 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5d72cff9[collection1] main]
[junit4] 2> 1303426 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 1303427 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 1303430 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1303430 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1623739923095879680
[junit4] 2> 1303433 INFO (searcherExecutor-2563-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 1303433 INFO (searcherExecutor-2563-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 1303433 INFO (searcherExecutor-2563-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 1303433 INFO (searcherExecutor-2563-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 1303437 INFO (searcherExecutor-2563-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 1303437 INFO (searcherExecutor-2563-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 1303437 INFO (searcherExecutor-2563-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 1303437 INFO (searcherExecutor-2563-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 1303438 INFO (searcherExecutor-2563-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 1303441 INFO (searcherExecutor-2563-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5d72cff9[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1303442 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=403657774, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 1303442 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 1303443 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5cea7a4c[collection1] main]
[junit4] 2> 1303453 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@2c2ecaf2
[junit4] 2> 1303453 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=2c2ecaf2
[junit4] 2> 1303453 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@375fd3d4: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@25b9fec6
[junit4] 2> 1303460 INFO (searcherExecutor-2563-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5cea7a4c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1303468 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=*:*&wt=xml} hits=0 status=0 QTime=2
[junit4] 2> 1303548 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.c.TestJmxIntegration After Reload: size of all core metrics: 443 MBeans: 442
[junit4] 2> 1303548 ERROR (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.c.TestJmxIntegration Unexpected bean names: []
[junit4] 2> 1303548 ERROR (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.c.TestJmxIntegration Unexpected metric names: [SEARCHER.searcher.maxDoc]
[junit4] 2> 1303549 INFO (TEST-TestJmxIntegration.testJmxOnCoreReload-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testJmxOnCoreReload
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestJmxIntegration -Dtests.method=testJmxOnCoreReload -Dtests.seed=31196B5ADAAA2ED6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.badapples=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=en-PH -Dtests.timezone=Etc/Greenwich -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] FAILURE 0.92s J0 | TestJmxIntegration.testJmxOnCoreReload <<<
[junit4] > Throwable #1: java.lang.AssertionError: Number of registered MBeans is not the same as the number of core metrics: 443 != 442
[junit4] > at __randomizedtesting.SeedInfo.seed([31196B5ADAAA2ED6:33A0B036A9E9F21B]:0)
[junit4] > at org.apache.solr.core.TestJmxIntegration.testJmxOnCoreReload(TestJmxIntegration.java:260)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1303582 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.SolrTestCaseJ4 ###Starting testJmxUpdate
[junit4] 2> 1303582 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
[junit4] 2> 1303582 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6c728831[collection1] realtime]
[junit4] 2> 1303583 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:* (-9223372036854775807)} 0 0
[junit4] 2> 1303583 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1623739923256311808,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1303583 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@5c1b1b0 commitCommandVersion:1623739923256311808
[junit4] 2> 1303583 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@10c4ff6c[collection1] main]
[junit4] 2> 1303583 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1303585 INFO (searcherExecutor-2563-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@10c4ff6c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1303585 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{commit=} 0 2
[junit4] 2> 1303609 INFO (TEST-TestJmxIntegration.testJmxUpdate-seed#[31196B5ADAAA2ED6]) [ x:collection1] o.a.s.c.TestJmxIntegration Mbeans in server: [solr_34558:dom1=solr,dom2=jvm,name0=system,name=properties, org.apache.logging.log4j2:type=2a139a55,component=StatusLogger, solr:dom1=core,dom2=collection1,category=QUERY,scope=/terms,name=serverErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/dismax,name=handlerStart, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Survivor-Space,name=max, java.lang:type=Threading, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mock,name=requests, solr:dom1=node,category=QUERY,scope=/admin/metrics/collector,name=handlerStart, solr:dom1=node,category=ADMIN,scope=/admin/metrics,name=requests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/mbeans,name=shardRequests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mock,name=errors, solr:dom1=jvm,name0=os,name=systemLoadAverage, solr:dom1=core,dom2=collection1,category=QUERY,scope=/get,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH1,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/defaults,name=serverErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-def,name=requestTimes, solr:dom1=core,dom2=collection1,category=UPDATE,scope=update,name=requestTimes, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak,name=shardRequests, solr:dom1=node,category=QUERY,scope=httpShardHandler,name=threadPool.httpShardExecutor.duration, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mock,name=handlerStart, solr:dom1=node,category=QUERY,scope=/admin/metrics/history,name=requests, solr:dom1=node,category=ADMIN,scope=/admin/metrics,name=serverErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update,name=handlerStart, solr:dom1=jvm,name0=buffers,name1=mapped,name=TotalCapacity, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update,name=totalTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/luke,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-def,name=timeouts, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=rollbacks, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=threadPool.recoveryExecutor.submitted, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=serverErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/stream,name=requestTimes, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/system,name=shardRequests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/dismax,name=clientErrors, solr:dom1=node,category=CONTAINER,scope=threadPool,name=coreContainerWorkExecutor.running, solr:dom1=core,dom2=collection1,category=QUERY,scope=/graph,name=clientErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/terms,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mltrh,name=shardRequests, solr:dom1=node,category=ADMIN,scope=/admin/collections,name=requestTimes, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=searcher,name=reader, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrFragmenter,name=gap.requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/export,name=requestTimes, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrBoundaryScanner,name=breakIterator.requests, solr:dom1=node,category=ADMIN,scope=/admin/authorization,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/debug/dump,name=shardRequests, org.apache.logging.log4j2:type=2a139a55,component=Loggers,name=org.eclipse.jetty, solr:dom1=node,category=ADMIN,scope=/admin/collections,name=handlerStart, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/segments,name=totalTime, solr:dom1=node,category=ADMIN,scope=/admin/autoscaling/history,name=clientErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/ping,name=errors, solr:dom1=node,category=QUERY,scope=/admin/metrics/history,name=shardRequests, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Survivor-Space,name=usage, solr:dom1=jvm,name0=memory,name1=pools,name2=Compressed-Class-Space,name=max, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/schema,name=clientErrors, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Old-Gen,name=init, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/health,name=shardRequests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/stream,name=timeouts, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=errors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/file,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/sql,name=requests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/logging,name=shardRequests, solr:dom1=core,dom2=collection1,category=TLOG,name=state, solr:dom1=core,dom2=collection1,category=CORE,scope=fs,name=spins, solr:dom1=jvm,name0=buffers,name1=direct,name=MemoryUsed, solr:dom1=jvm,name0=classes,name=unloaded, solr:dom1=jvm,name0=threads,name1=waiting,name=count, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-def,name=errors, solr_44020:dom1=solr,dom2=jvm,name0=system,name=properties, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=threadPool.parallelCoreAdminExecutor.submitted, solr:dom1=core,dom2=collection1,category=TLOG,scope=replay,name=remaining.bytes, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/properties,name=timeouts, solr:dom1=node,category=CONTAINER,scope=cores,name=loaded, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/schema,name=handlerStart, solr:dom1=node,category=ADMIN,scope=/admin/autoscaling/history,name=timeouts, solr:dom1=node,category=ADMIN,scope=/admin/authorization,name=handlerStart, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/luke,name=shardRequests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mltrh,name=totalTime, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update,name=clientErrors, solr_35452:dom1=solr,dom2=jvm,name0=system,name=properties, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-invariants,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/export,name=totalTime, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=new,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/sql,name=timeouts, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/config,name=clientErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/csv,name=handlerStart, solr:dom1=node,category=QUERY,scope=httpShardHandler,name=threadPool.httpShardExecutor.completed, solr:dom1=core,dom2=collection1,category=QUERY,scope=/debug/dump,name=clientErrors, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/graph,name=serverErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak_Direct,name=serverErrors, java.lang:type=Runtime, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=cumulativeDeletesById, solr:dom1=core,dom2=collection1,category=QUERY,scope=/defaults,name=timeouts, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=cumulativeDeletesByQuery, solr:dom1=core,dom2=collection1,category=QUERY,scope=/dismax,name=timeouts, solr:dom1=node,category=ADMIN,scope=/admin/authorization,name=errors, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=threadPool.recoveryExecutor.duration, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak_Direct,name=timeouts, solr:dom1=core,dom2=collection1,category=CORE,scope=fs,name=totalSpace, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH_Direct,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/terms,name=clientErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/mbeans,name=serverErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-def,name=handlerStart, solr:dom1=node,category=CONTAINER,scope=threadPool,name=coreContainerWorkExecutor.duration, solr_34426:dom1=solr,dom2=jvm,name0=system,name=properties, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak_Direct,name=requests, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper,name=clientErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/segments,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak_Direct,name=clientErrors, solr_41390:dom1=solr,dom2=jvm,name0=system,name=properties, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=threadPool.updateOnlyExecutor.running, solr:dom1=node,category=CONTAINER,scope=cores,name=unloaded, solr:dom1=node,category=ADMIN,scope=/admin/metrics,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-invariants,name=serverErrors, java.lang:type=MemoryPool,name=PS Eden Space, solr:dom1=node,category=ADMIN,scope=/admin/metrics,name=timeouts, solr:dom1=core,dom2=collection1,category=CORE,scope=fs,name=usableSpace, solr:dom1=jvm,name0=os,name=totalPhysicalMemorySize, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH,name=serverErrors, solr:dom1=jvm,name0=os,name=totalSwapSpaceSize, solr:dom1=jvm,name0=memory,name1=pools,name2=Code-Cache,name=init, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=cumulativeErrors, java.lang:type=MemoryPool,name=PS Survivor Space, solr:dom1=node,category=QUERY,scope=/admin/metrics/collector,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mlt,name=handlerStart, solr:dom1=jvm,name0=os,name=availableProcessors, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=threadPool.recoveryExecutor.running, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak,name=timeouts, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak,name=errors, solr:dom1=jvm,name0=memory,name1=pools,name2=Compressed-Class-Space,name=used, solr:dom1=node,category=CONTAINER,scope=fs,name=path, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper,name=timeouts, solr:dom1=jvm,name0=threads,name=count, solr:dom1=node,category=ADMIN,scope=/admin/authorization,name=timeouts, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=searcher,name=caching, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper/status,name=serverErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mock,name=serverErrors, solr:dom1=jvm,name0=memory,name1=total,name=used, solr:dom1=jvm,name0=memory,name1=pools,name2=Code-Cache,name=usage, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrFormatter,name=html.requests, solr:dom1=node,category=CONTAINER,scope=fs,name=spins, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=clientErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=deletesByQuery, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/schema,name=timeouts, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json/docs,name=serverErrors, solr:dom1=jvm,name0=threads,name1=new,name=count, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak,name=handlerStart, solr:dom1=core,dom2=collection1,category=TLOG,scope=replay,name=ops, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update,name=shardRequests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH1,name=handlerStart, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/threads,name=serverErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/logging,name=serverErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/ping,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/export,name=timeouts, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Old-Gen,name=committed, solr:dom1=node,category=ADMIN,scope=/admin/autoscaling/history,name=totalTime, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=searcher,name=openedAt, solr:dom1=core,dom2=collection1,category=QUERY,scope=/stream,name=clientErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mlt,name=requestTimes, solr:dom1=node,category=ADMIN,scope=/admin/configs,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mock,name=initCount, solr:dom1=core,dom2=collection1,category=UPDATE,scope=update,name=errors, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper/status,name=totalTime, solr:dom1=node,category=QUERY,scope=/admin/metrics/history,name=clientErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=update,name=clientErrors, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=threadPool.updateOnlyExecutor.duration, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak,name=requests, org.apache.logging.log4j2:type=2a139a55,component=Loggers,name=org.apache.directory, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/select,name=requests, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=softAutoCommits, solr:dom1=jvm,name0=memory,name1=non-heap,name=used, org.apache.logging.log4j2:type=2a139a55,component=Appenders,name=STDERR, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper/status,name=timeouts, solr:dom1=node,category=ADMIN,scope=/admin/metrics,name=handlerStart, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/segments,name=serverErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=update,name=shardRequests, solr:dom1=node,category=ADMIN,scope=/admin/collections,name=clientErrors, java.lang:type=MemoryPool,name=Code Cache, solr:dom1=core,dom2=collection1,category=CORE,name=refCount, solr_44904:dom1=solr,dom2=jvm,name0=system,name=properties, java.nio:type=BufferPool,name=direct, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json,name=totalTime, solr:dom1=node,category=ADMIN,scope=/admin/collections,name=errors, solr:dom1=jvm,name0=threads,name1=daemon,name=count, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper,name=serverErrors, solr:dom1=node,category=CONTAINER,scope=fs,name=coreRoot.path, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/segments,name=errors, solr:dom1=jvm,name0=memory,name1=non-heap,name=init, solr:dom1=node,category=ADMIN,scope=/admin/configs,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/sql,name=requestTimes, solr:dom1=core,dom2=collection1,category=CACHE,scope=searcher,name=queryResultCache, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-invariants,name=totalTime, solr:dom1=node,category=ADMIN,scope=/admin/autoscaling/history,name=handlerStart, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=searcher,name=searcherName, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH_Direct,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/dismax,name=requests, solr:dom1=jvm,name0=memory,name1=heap,name=max, solr:dom1=core,dom2=collection1,category=QUERY,scope=/defaults,name=shardRequests, solr:dom1=node,category=CONTAINER,scope=threadPool,name=coreLoadExecutor.duration, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/csv,name=shardRequests, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=serverErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=update,name=timeouts, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/luke,name=errors, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=requests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/mbeans,name=requestTimes, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/file,name=timeouts, solr:dom1=core,dom2=c
[junit4] 2> ollection1,category=REPLICATION,scope=/replication,name=indexSize, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/ping,name=shardRequests, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper,name=errors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/ping,name=timeouts, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/graph,name=requests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/schema,name=shardRequests, solr:dom1=node,category=ADMIN,scope=/admin/configs,name=shardRequests, solr_46471:dom1=solr,dom2=jvm,name0=system,name=properties, solr:dom1=jvm,name0=buffers,name1=mapped,name=Count, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/schema,name=requestTimes, solr:dom1=core,dom2=collection1,category=QUERY,scope=/defaults,name=errors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/segments,name=requests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/file,name=shardRequests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-invariants,name=timeouts, solr:dom1=core,dom2=collection1,category=UPDATE,scope=update,name=totalTime, solr:dom1=core,dom2=collection1,category=UPDATE,scope=update,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH1,name=clientErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json/docs,name=clientErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/properties,name=serverErrors, java.lang:type=Compilation, solr:dom1=jvm,name0=memory,name1=pools,name2=Compressed-Class-Space,name=committed, solr:dom1=node,category=ADMIN,scope=/admin/autoscaling/history,name=requests, solr:dom1=node,category=QUERY,scope=/admin/metrics/collector,name=serverErrors, solr:dom1=core,dom2=collection1,category=TLOG,scope=buffered,name=ops, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=timeouts, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=adds, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/properties,name=errors, solr_45662:dom1=solr,dom2=jvm,name0=system,name=properties, solr:dom1=jvm,name0=os,name=version, solr:dom1=core,dom2=collection1,category=UPDATE,scope=update,name=serverErrors, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=pendingConnections, solr:dom1=jvm,name0=memory,name1=pools,name2=Metaspace,name=usage, solr:dom1=core,dom2=collection1,category=CACHE,scope=searcher,name=documentCache, solr:dom1=node,category=ADMIN,scope=/admin/authorization,name=shardRequests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/plugins,name=totalTime, solr_43294:dom1=solr,dom2=jvm,name0=system,name=properties, solr:dom1=core,dom2=collection1,category=QUERY,scope=/graph,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mlt,name=errors, solr_39351:dom1=solr,dom2=jvm,name0=system,name=properties, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak_Direct,name=handlerStart, java.lang:type=MemoryPool,name=Compressed Class Space, solr:dom1=core,dom2=collection1,category=QUERY,scope=/terms,name=requestTimes, solr_40605:dom1=solr,dom2=jvm,name0=system,name=properties, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/schema,name=errors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH_Direct,name=totalTime, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/csv,name=clientErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/properties,name=totalTime, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/csv,name=requests, solr:dom1=jvm,name0=os,name=committedVirtualMemorySize, solr:dom1=core,dom2=collection1,category=QUERY,scope=/tvrh,name=errors, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Old-Gen,name=usage, solr:dom1=core,dom2=collection1,category=QUERY,scope=/terms,name=requests, solr:dom1=jvm,name0=memory,name1=non-heap,name=max, solr:dom1=jvm,name0=os,name=maxFileDescriptorCount, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrFragmentsBuilder,name=simple.requests, solr:dom1=core,dom2=collection1,category=CORE,name=coreName, solr:dom1=node,category=CONTAINER,scope=threadPool,name=coreLoadExecutor.running, solr:dom1=core,dom2=collection1,category=QUERY,scope=/sql,name=handlerStart, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/health,name=handlerStart, solr:dom1=jvm,name0=memory,name1=heap,name=used, solr:dom1=core,dom2=collection1,category=CACHE,scope=searcher,name=perSegFilter, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/logging,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/sql,name=errors, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=confFilesToReplicate, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json/docs,name=requests, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/luke,name=totalTime, solr:dom1=node,category=CONTAINER,scope=fs,name=coreRoot.usableSpace, solr:dom1=jvm,name0=os,name=systemCpuLoad, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrFragListBuilder,name=default.requests, solr:dom1=node,category=ADMIN,scope=/admin/info,name=shardRequests, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=splits, solr:dom1=node,category=UPDATE,scope=updateShardHandler,name=threadPool.updateOnlyExecutor.completed, solr:dom1=core,dom2=collection1,category=QUERY,scope=/terms,name=handlerStart, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=fetcher, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH_Direct,name=requestTimes, solr:dom1=jvm,name0=threads,name1=blocked,name=count, solr:dom1=core,dom2=collection1,category=QUERY,scope=/graph,name=timeouts, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper,name=requestTimes, solr:dom1=node,category=ADMIN,scope=/admin/autoscaling/history,name=shardRequests, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=totalTime, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrBoundaryScanner,name=default.requests, solr:dom1=jvm,name0=gc,name1=PS-MarkSweep,name=time, solr:dom1=node,category=ADMIN,scope=/admin/collections,name=requests, solr:dom1=node,category=ADMIN,scope=/admin/configs,name=totalTime, solr:dom1=core,dom2=collection1,category=TLOG,scope=applyingBuffered,name=ops, solr_45632:dom1=solr,dom2=jvm,name0=system,name=properties, solr:dom1=node,category=ADMIN,scope=/admin/info,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-def,name=totalTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/health,name=requestTimes, solr:dom1=jvm,name0=buffers,name1=direct,name=TotalCapacity, solr:dom1=node,category=ADMIN,scope=/admin/autoscaling/history,name=serverErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/properties,name=clientErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=update,name=handlerStart, solr_35997:dom1=solr,dom2=jvm,name0=system,name=properties, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=generation, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/csv,name=timeouts, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/health,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mltrh,name=errors, java.lang:type=OperatingSystem, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/luke,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/tvrh,name=handlerStart, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/threads,name=totalTime, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json/docs,name=timeouts, solr:dom1=node,category=ADMIN,scope=/admin/metrics,name=shardRequests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mock,name=clientErrors, solr:dom1=jvm,name0=threads,name1=terminated,name=count, java.lang:type=MemoryManager,name=Metaspace Manager, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/mbeans,name=handlerStart, solr:dom1=node,category=QUERY,scope=/admin/metrics/collector,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH1,name=shardRequests, solr:dom1=node,category=QUERY,scope=/admin/metrics/collector,name=shardRequests, solr:dom1=node,category=ADMIN,scope=/admin/metrics,name=totalTime, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH1,name=errors, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper,name=handlerStart, solr_36079:dom1=solr,dom2=jvm,name0=system,name=properties, solr:dom1=core,dom2=collection1,category=TLOG,scope=replay,name=remaining.logs, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/system,name=requestTimes, solr:dom1=core,dom2=collection1,category=CORE,scope=fs,name=path, solr:dom1=node,category=CONTAINER,scope=threadPool,name=coreContainerWorkExecutor.completed, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/properties,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/sql,name=serverErrors, solr_46395:dom1=solr,dom2=jvm,name0=system,name=properties, solr:dom1=core,dom2=collection1,category=QUERY,scope=/sql,name=totalTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/logging,name=totalTime, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=threadPool.parallelCoreAdminExecutor.duration, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/luke,name=serverErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH,name=totalTime, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json/docs,name=totalTime, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Eden-Space,name=max, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Old-Gen,name=used, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=shardRequests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckWithWordbreak,name=serverErrors, solr:dom1=core,dom2=collection1,category=INDEX,name=sizeInBytes, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper/status,name=shardRequests, solr:dom1=jvm,name0=memory,name1=pools,name2=Metaspace,name=max, solr:dom1=core,dom2=collection1,category=QUERY,scope=/tvrh,name=shardRequests, solr:dom1=node,category=ADMIN,scope=/admin/info,name=requestTimes, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=commits, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=isSlave, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/threads,name=requests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/sql,name=clientErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/schema,name=serverErrors, solr:dom1=core,dom2=collection1,category=UPDATE,scope=updateHandler,name=cumulativeAdds, solr:dom1=node,category=ADMIN,scope=/admin/authorization,name=clientErrors, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/segments,name=shardRequests, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mock,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mlt,name=serverErrors, solr:dom1=core,dom2=collection1,category=QUERY,scope=/select,name=timeouts, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/mbeans,name=timeouts, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/logging,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mltrh,name=clientErrors, solr:dom1=jvm,name0=os,name=freeSwapSpaceSize, solr:dom1=core,dom2=collection1,category=QUERY,scope=/get,name=totalTime, java.lang:type=MemoryManager,name=CodeCacheManager, solr:dom1=core,dom2=collection1,category=QUERY,scope=/stream,name=requests, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=indexPath, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/threads,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/debug/dump,name=requestTimes, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Eden-Space,name=committed, solr:dom1=core,dom2=collection1,category=QUERY,scope=/export,name=requests, solr:dom1=node,category=ADMIN,scope=/admin/configs,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/debug/dump,name=totalTime, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=requestTimes, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Old-Gen,name=used-after-gc, solr:dom1=node,category=ADMIN,scope=/admin/collections,name=timeouts, java.lang:type=GarbageCollector,name=PS Scavenge, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/system,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/graph,name=totalTime, solr:dom1=node,category=ADMIN,scope=/admin/info,name=clientErrors, solr:dom1=jvm,name0=memory,name1=heap,name=usage, solr:dom1=core,dom2=collection1,category=INDEX,name=size, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/plugins,name=requestTimes, java.nio:type=BufferPool,name=mapped, solr:dom1=jvm,name0=memory,name1=pools,name2=PS-Eden-Space,name=used-after-gc, com.sun.management:type=DiagnosticCommand, solr:dom1=core,dom2=collection1,category=REPLICATION,scope=/replication,name=replicationEnabled, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/config,name=serverErrors, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=new,name=maxReached, solr:dom1=node,category=ADMIN,scope=/admin/info,name=totalTime, solr:dom1=core,dom2=collection1,category=ADMIN,scope=/admin/plugins,name=clientErrors, solr:dom1=node,category=ADMIN,scope=/admin/cores,name=threadPool.parallelCoreAdminExecutor.running, org.apache.logging.log4j2:type=2a139a55,component=Loggers,name=org.apache.hadoop, solr:dom1=node,category=ADMIN,scope=/admin/zookeeper/status,name=requests, solr:dom1=jvm,name0=os,name=arch, solr_33523:dom1=solr,dom2=jvm,name0=system,name=properties, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/csv,name=serverErrors, solr:dom1=node,category=QUERY,scope=/admin/metrics/history,name=timeouts, solr:dom1=core,dom2=collection1,category=QUERY,scope=/mock,name=shardRequests, solr:dom1=node,category=QUERY,scope=/admin/metrics/history,name=handlerStart, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-def,name=requests, solr:dom1=jvm,name0=memory,name1=pools,name2=Compressed-Class-Space,name=init, solr:dom1=jvm,name0=os,name=name, solr:dom1=core,dom2=collection1,category=QUERY,scope=/search-facet-def,name=serverErrors, solr:dom1=core,dom2=collection1,category=SEARCHER,scope=searcher,name=maxDoc, solr:dom1=jvm,name0=os,name=processCpuLoad, solr:dom1=core,dom2=collection1,category=UPDATE,scope=/update/json,name=errors, solr:dom1=jvm,name0=memory,name1=pools,name2=Metaspace,name=committed, solr:dom1=core,dom2=collection1,category=HIGHLIGHTER,scope=SolrFragmentsBuilder,name=scoreOrder.requests, solr:dom1=node,category=CONTAINER,scope=fs,name=usableSpace, solr:dom1=core,dom2=collection1,category=QUERY,scope=/spellCheckCompRH1,name=serverErrors, solr:dom1=node,category=A
[...truncated too long message...]
.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] 2> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] 2> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] 2> at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
[junit4] 2> at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
[junit4] 2> at org.eclipse.jetty.io.ManagedSelector$$Lambda$41/621753608.run(Unknown Source)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$10/916963197.run(Unknown Source)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene80): {}, docValues:{}, maxPointsInLeafNode=2021, maxMBSortInHeap=5.0220498055603935, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@5acf4961), locale=nl-BE, timezone=Europe/Ulyanovsk
[junit4] 2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=565,free=51552968,total=486539264
[junit4] 2> NOTE: All tests run in this JVM: [TestRequestStatusCollectionAPI, LeaderFailoverAfterPartitionTest, URLClassifyProcessorTest, PeerSyncWithLeaderTest, ConvertedLegacyTest, MoveReplicaHDFSTest, DistributedFacetPivotLargeTest, ExternalFileFieldSortTest, TestConfigSetImmutable, TestStressInPlaceUpdates, TestManagedSchema, TestHashQParserPlugin, TestXIncludeConfig, SchemaApiFailureTest, LeaderElectionContextKeyTest, InfoHandlerTest, TestQueryTypes, PropertiesRequestHandlerTest, SpellCheckComponentTest, TestSolrDeletionPolicy2, ScheduledTriggerIntegrationTest, AssignBackwardCompatibilityTest, HdfsUnloadDistributedZkTest, TestDistributedGrouping, LeaderVoteWaitTimeoutTest, TestLegacyFieldCache, TestFacetMethods, TestFieldCache, SearchRateTriggerTest, TestMiniSolrCloudClusterSSL, TestSSLRandomization, HdfsChaosMonkeyNothingIsSafeTest, TestReqParamsAPI, SortByFunctionTest, CdcrUpdateLogTest, TestSolrFieldCacheBean, BasicAuthIntegrationTest, TestCustomStream, TestLegacyNumericUtils, TestStressRecovery, CdcrRequestHandlerTest, TestWithCollection, AutoScalingHandlerTest, DistributedIntervalFacetingTest, RandomizedTaggerTest, OverriddenZkACLAndCredentialsProvidersTest, TestHighFrequencyDictionaryFactory, TransactionLogTest, ImplicitSnitchTest, ExplicitHLLTest, TestSimTriggerIntegration, IgnoreCommitOptimizeUpdateProcessorFactoryTest, ActionThrottleTest, TestAuthenticationFramework, RegexBytesRefFilterTest, TestCloudSearcherWarming, TestHdfsCloudBackupRestore, JWTAuthPluginTest, HdfsSyncSliceTest, TestRestManager, CreateRoutedAliasTest, TestRandomCollapseQParserPlugin, TestLuceneIndexBackCompat, TestCSVLoader, TestDynamicFieldCollectionResource, TestSolr4Spatial, XmlUpdateRequestHandlerTest, TestDeleteCollectionOnDownNodes, BigEndianAscendingWordDeserializerTest, VersionInfoTest, TestZkAclsWithHadoopAuth, HLLUtilTest, SignificantTermsQParserPluginTest, TestDownShardTolerantSearch, SolrMetricsIntegrationTest, CoreSorterTest, TestRemoteStreaming, TestReload, HealthCheckHandlerTest, OverseerTest, TestChildDocTransformerHierarchy, TestCloudJSONFacetJoinDomain, LargeFieldTest, DistributedQueryComponentCustomSortTest, TestMaxScoreQueryParser, TestCSVResponseWriter, TestSimpleTrackingShardHandler, TestJsonFacets, TestIndexingPerformance, TestFileDictionaryLookup, RecoveryZkTest, TestSafeXMLParsing, CoreAdminOperationTest, DateMathParserTest, TestNumericRangeQuery32, TestNestedDocsSort, ByteBuffersDirectoryFactoryTest, DistributedFacetPivotSmallTest, TestPartialUpdateDeduplication, TestLeaderElectionWithEmptyReplica, TestSchemaVersionResource, TestPushWriter, SimplePostToolTest, UpdateRequestProcessorFactoryTest, QueryResultKeyTest, BJQParserTest, MetricTriggerIntegrationTest, ConfigSetsAPITest, SearchHandlerTest, HdfsRecoverLeaseTest, SpatialRPTFieldTypeTest, SolrCoreTest, HdfsLockFactoryTest, JSONWriterTest, TriggerCooldownIntegrationTest, TestStressVersions, TestSizeLimitedDistributedMap, SolrRequestParserTest, CustomCollectionTest, ChangedSchemaMergeTest, TestPerFieldSimilarity, TestManagedStopFilterFactory, BadCopyFieldTest, NodeAddedTriggerTest, TestApiFramework, PreAnalyzedFieldTest, TestSchemalessBufferedUpdates, AssignTest, TemplateUpdateProcessorTest, HDFSCollectionsAPITest, TestSQLHandlerNonCloud, SolrIndexSplitterTest, TestTrie, TestMergePolicyConfig, SolrCoreMetricManagerTest, AnalysisAfterCoreReloadTest, BasicFunctionalityTest, CursorPagingTest, DisMaxRequestHandlerTest, OutputWriterTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestDocumentBuilder, TestJoin, TestSolrCoreProperties, TestTolerantSearch, TestCharFilters, TestReversedWildcardFilterFactory, TestWordDelimiterFilterFactory, TokenizerChainTest, TestEmbeddedSolrServerSchemaAPI, BasicDistributedZkTest, BasicZkTest, ChaosMonkeyNothingIsSafeTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, ChaosMonkeySafeLeaderTest, ChaosMonkeySafeLeaderWithPullReplicasTest, ConnectionManagerTest, DeleteNodeTest, DeleteShardTest, DeleteStatusTest, DistribJoinFromCollectionTest, DistributedQueueTest, DistributedVersionInfoTest, ForceLeaderTest, HttpPartitionOnCommitTest, HttpPartitionTest, LeaderFailureAfterFreshStartTest, LeaderTragicEventTest, LegacyCloudClusterPropTest, MetricsHistoryIntegrationTest, MigrateRouteKeyTest, MultiThreadedOCPTest, NodeMutatorTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerRolesTest, OverseerStatusTest, OverseerTaskQueueTest, RemoteQueryErrorTest, ReplaceNodeNoTargetTest, ReplaceNodeTest, ReplicationFactorTest, RestartWhileUpdatingTest, RollingRestartTest, ShardRoutingCustomTest, ShardRoutingTest, SharedFSAutoReplicaFailoverTest, SliceStateTest, SolrXmlInZkTest, TestCloudInspectUtil, TestCloudPseudoReturnFields, TestClusterProperties, TestConfigSetsAPI, TestConfigSetsAPIExclusivity, TestConfigSetsAPIZkFailure, TestCryptoKeys, TestDistribDocBasedVersion, TestDistributedMap, TestLeaderElectionZkExpiry, TestLockTree, TestOnReconnectListenerSupport, TestPrepRecovery, TestPullReplica, TestRandomRequestDistribution, TestRebalanceLeaders, TestRequestForwarding, TestSolrCloudWithKerberosAlt, TestSolrCloudWithSecureImpersonation, TestStressCloudBlindAtomicUpdates, TestTolerantUpdateProcessorCloud, TestTolerantUpdateProcessorRandomCloud, TestUtilizeNode, TestZkChroot, TlogReplayBufferedWhileIndexingTest, TriLevelCompositeIdRoutingTest]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TriLevelCompositeIdRoutingTest -Dtests.seed=31196B5ADAAA2ED6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.badapples=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=nl-BE -Dtests.timezone=Europe/Ulyanovsk -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 0.00s J2 | TriLevelCompositeIdRoutingTest (suite) <<<
[junit4] > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
[junit4] > at __randomizedtesting.SeedInfo.seed([31196B5ADAAA2ED6]:0)
[junit4] Completed [842/842 (5!)] on J2 in 7233.32s, 1 test, 2 errors <<< FAILURES!
[...truncated 1 lines...]
[junit4] JVM J2: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20190126_154917_1152879976612986033288.sysout
[junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/heapdumps/java_pid10040.hprof ...
[junit4] Heap dump file created [791836890 bytes in 3.581 secs]
[junit4] <<< JVM J2: EOF ----
[junit4] JVM J2: stderr was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20190126_154917_1151320486448861748079.syserr
[junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
[junit4]
[junit4] Exception: java.lang.IllegalArgumentException thrown from the UncaughtExceptionHandler in thread "Scheduler-711239499"
[junit4] <<< JVM J2: EOF ----
[...truncated 8928 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/build.xml:661: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid10040.hprof
* java_pid10047.hprof
Total time: 747 minutes 19 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-BadApples-NightlyTests-master - Build # 46 -
Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-master/46/
2 tests failed.
FAILED: org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest.test
Error Message:
There are still nodes recoverying - waited for 320 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
at __randomizedtesting.SeedInfo.seed([1B990FFF714B6410:93CD3025DFB709E8]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:195)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1038)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1595)
at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:144)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.RestartWhileUpdatingTest.test
Error Message:
There are still nodes recoverying - waited for 320 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
at __randomizedtesting.SeedInfo.seed([1B990FFF714B6410:93CD3025DFB709E8]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:195)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1038)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1595)
at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:144)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 14032 lines...]
[junit4] JVM J2: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20190119_151432_8282816604587647437958.sysout
[junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/heapdumps/java_pid21223.hprof ...
[junit4] Heap dump file created [852285094 bytes in 5.555 secs]
[junit4] <<< JVM J2: EOF ----
[junit4] JVM J2: stderr was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20190119_151432_8282876645628650944136.syserr
[junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
[junit4] WARN: Unhandled exception in event serialization. -> java.lang.OutOfMemoryError: GC overhead limit exceeded
[junit4] <<< JVM J2: EOF ----
[...truncated 190 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/init-core-data-001
[junit4] 2> 1977144 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=8 numCloses=8
[junit4] 2> 1977144 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 1977148 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 1977148 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 1977149 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /c/u
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 1977203 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 1977224 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1977226 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 1977267 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_42614_hdfs____vtikat/webapp
[junit4] 2> 1977741 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42614
[junit4] 2> 1977857 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1977858 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 1977880 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_43474_datanode____9lnyft/webapp
[junit4] 2> 1978460 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43474
[junit4] 2> 1978543 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1978544 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 1978570 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_41327_datanode____ssydfm/webapp
[junit4] 2> 1978631 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:42794) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 1978651 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x2024c322aa56d7: from storage DS-fcdb5164-8e56-408d-b41b-13b462a1de42 node DatanodeRegistration(127.0.0.1:33272, datanodeUuid=23bb5fc7-7bac-43f8-88a6-f2e6a643953e, infoPort=44184, infoSecurePort=0, ipcPort=41895, storageInfo=lv=-56;cid=testClusterID;nsid=1734812269;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 1978651 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x2024c322aa56d7: from storage DS-1c36d9fc-9012-44f6-959a-036d719bf1eb node DatanodeRegistration(127.0.0.1:33272, datanodeUuid=23bb5fc7-7bac-43f8-88a6-f2e6a643953e, infoPort=44184, infoSecurePort=0, ipcPort=41895, storageInfo=lv=-56;cid=testClusterID;nsid=1734812269;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 1979089 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[1B990FFF714B6410]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41327
[junit4] 2> 1979216 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:42794) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 1979229 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x2024c3452bcd0e: from storage DS-ee19c920-1598-4fae-b708-cc8f50a3864c node DatanodeRegistration(127.0.0.1:35245, datanodeUuid=390638e6-530b-4ae5-af7a-99fe6464d780, infoPort=46046, infoSecurePort=0, ipcPort=34870, storageInfo=lv=-56;cid=testClusterID;nsid=1734812269;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 1979230 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x2024c3452bcd0e: from storage DS-6b89699f-0721-4b6b-99d1-d818de742c45 node DatanodeRegistration(127.0.0.1:35245, datanodeUuid=390638e6-530b-4ae5-af7a-99fe6464d780, infoPort=46046, infoSecurePort=0, ipcPort=34870, storageInfo=lv=-56;cid=testClusterID;nsid=1734812269;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 1979504 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1979504 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1979504 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1979604 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.ZkTestServer start zk server on port:40686
[junit4] 2> 1979604 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:40686
[junit4] 2> 1979604 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 40686
[junit4] 2> 1979609 INFO (zkConnectionManagerCallback-4656-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1979614 INFO (zkConnectionManagerCallback-4658-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1979616 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1979617 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 1979618 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1979620 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1979622 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1979623 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1979624 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1979627 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1979628 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1979629 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1979630 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1979634 INFO (zkConnectionManagerCallback-4662-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1979637 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 1979820 WARN (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 1979821 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 1979821 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 1979821 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 1979825 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1979825 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1979825 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 1979828 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3395b730{/c/u,null,AVAILABLE}
[junit4] 2> 1979829 WARN (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@62072312[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1979829 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5251f1cd{SSL,[ssl, http/1.1]}{127.0.0.1:36572}
[junit4] 2> 1979829 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.e.j.s.Server Started @1979896ms
[junit4] 2> 1979829 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:42794/hdfs__localhost_42794__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-BadApples-NightlyTests-master_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001_tempDir-002_control_data, hostContext=/c/u, hostPort=36572, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/control-001/cores}
[junit4] 2> 1979830 ERROR (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1979830 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1979830 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 1979830 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1979830 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1979830 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-01-19T15:47:32.810Z
[junit4] 2> 1979832 INFO (zkConnectionManagerCallback-4664-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1979834 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1979834 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/control-001/solr.xml
[junit4] 2> 1979839 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1979839 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1979841 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1980210 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 1980215 WARN (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 1980221 WARN (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@4d6e1650[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1980227 WARN (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 1980228 WARN (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@424e2867[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1980229 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40686/solr
[junit4] 2> 1980231 INFO (zkConnectionManagerCallback-4670-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1980234 INFO (zkConnectionManagerCallback-4672-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1980322 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [n:127.0.0.1:36572_c%2Fu ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:36572_c%2Fu
[junit4] 2> 1980323 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [n:127.0.0.1:36572_c%2Fu ] o.a.s.c.Overseer Overseer (id=72650538903797765-127.0.0.1:36572_c%2Fu-n_0000000000) starting
[junit4] 2> 1980330 INFO (zkConnectionManagerCallback-4679-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1980333 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [n:127.0.0.1:36572_c%2Fu ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40686/solr ready
[junit4] 2> 1980334 INFO (OverseerStateUpdate-72650538903797765-127.0.0.1:36572_c%2Fu-n_0000000000) [n:127.0.0.1:36572_c%2Fu ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:36572_c%2Fu
[junit4] 2> 1980334 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [n:127.0.0.1:36572_c%2Fu ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36572_c%2Fu
[junit4] 2> 1980336 INFO (OverseerStateUpdate-72650538903797765-127.0.0.1:36572_c%2Fu-n_0000000000) [n:127.0.0.1:36572_c%2Fu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1980337 INFO (zkCallback-4678-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1980365 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [n:127.0.0.1:36572_c%2Fu ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1980396 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [n:127.0.0.1:36572_c%2Fu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea
[junit4] 2> 1980411 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [n:127.0.0.1:36572_c%2Fu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea
[junit4] 2> 1980411 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [n:127.0.0.1:36572_c%2Fu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea
[junit4] 2> 1980412 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [n:127.0.0.1:36572_c%2Fu ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/control-001/cores
[junit4] 2> 1980452 INFO (zkConnectionManagerCallback-4685-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1980454 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1980455 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40686/solr ready
[junit4] 2> 1980479 INFO (qtp495006951-63416) [n:127.0.0.1:36572_c%2Fu ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:36572_c%252Fu&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1980486 INFO (OverseerThreadFactory-4786-thread-1-processing-n:127.0.0.1:36572_c%2Fu) [n:127.0.0.1:36572_c%2Fu ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 1980598 INFO (qtp495006951-63415) [n:127.0.0.1:36572_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1980599 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1980600 INFO (qtp495006951-63413) [n:127.0.0.1:36572_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1980600 INFO (qtp495006951-63414) [n:127.0.0.1:36572_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1980601 INFO (qtp495006951-63415) [n:127.0.0.1:36572_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1980611 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1980612 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1981639 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 1981657 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 1981760 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1981782 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 1981783 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea
[junit4] 2> 1981783 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 1981788 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=97.541015625, floorSegmentMB=1.357421875, forceMergeDeletesPctAllowed=11.115188937090268, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5987367564478258, deletesPctAllowed=33.419722422116344
[junit4] 2> 1981799 WARN (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1981886 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1981886 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1981890 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 30000ms;
[junit4] 2> 1981890 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
[junit4] 2> 1981892 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=8, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.17040485074340178]
[junit4] 2> 1981893 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@bfb0c30[control_collection_shard1_replica_n1] main]
[junit4] 2> 1981894 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1981895 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1981896 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1981896 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1623104269714456576
[junit4] 2> 1981900 INFO (searcherExecutor-4791-thread-1-processing-n:127.0.0.1:36572_c%2Fu x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@bfb0c30[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1981908 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 1981908 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
[junit4] 2> 1981912 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1981912 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1981912 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:36572/c/u/control_collection_shard1_replica_n1/
[junit4] 2> 1981913 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1981913 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:36572/c/u/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 1981913 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72650538903797765-core_node2-n_0000000000
[junit4] 2> 1981916 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:36572/c/u/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 1982018 INFO (zkCallback-4671-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1982018 INFO (zkCallback-4671-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1982021 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1982025 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1413
[junit4] 2> 1982029 INFO (qtp495006951-63416) [n:127.0.0.1:36572_c%2Fu ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 1982125 INFO (zkCallback-4671-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1982125 INFO (zkCallback-4671-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1982125 INFO (zkCallback-4671-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1982126 INFO (qtp495006951-63416) [n:127.0.0.1:36572_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:36572_c%252Fu&wt=javabin&version=2} status=0 QTime=1647
[junit4] 2> 1982133 INFO (zkConnectionManagerCallback-4691-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1982135 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1982136 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40686/solr ready
[junit4] 2> 1982136 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1982139 INFO (qtp495006951-63416) [n:127.0.0.1:36572_c%2Fu ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1982142 INFO (OverseerThreadFactory-4786-thread-2-processing-n:127.0.0.1:36572_c%2Fu) [n:127.0.0.1:36572_c%2Fu ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 1982143 INFO (OverseerCollectionConfigSetProcessor-72650538903797765-127.0.0.1:36572_c%2Fu-n_0000000000) [n:127.0.0.1:36572_c%2Fu ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1982346 WARN (OverseerThreadFactory-4786-thread-2-processing-n:127.0.0.1:36572_c%2Fu) [n:127.0.0.1:36572_c%2Fu ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 1982349 INFO (qtp495006951-63416) [n:127.0.0.1:36572_c%2Fu ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 1982350 INFO (qtp495006951-63416) [n:127.0.0.1:36572_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=211
[junit4] 2> 1982352 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=3
[junit4] 2> 1982570 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/shard-1-001 of type NRT
[junit4] 2> 1982578 WARN (closeThreadPool-4692-thread-1) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 1982578 INFO (closeThreadPool-4692-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 1982578 INFO (closeThreadPool-4692-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 1982578 INFO (closeThreadPool-4692-thread-1) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 1982579 INFO (closeThreadPool-4692-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1982579 INFO (closeThreadPool-4692-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1982579 INFO (closeThreadPool-4692-thread-1) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 1982580 INFO (closeThreadPool-4692-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2fe06f73{/c/u,null,AVAILABLE}
[junit4] 2> 1982580 WARN (closeThreadPool-4692-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@3a1a47a6[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1982581 INFO (closeThreadPool-4692-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@46541373{SSL,[ssl, http/1.1]}{127.0.0.1:35515}
[junit4] 2> 1982581 INFO (closeThreadPool-4692-thread-1) [ ] o.e.j.s.Server Started @1982647ms
[junit4] 2> 1982581 INFO (closeThreadPool-4692-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:42794/hdfs__localhost_42794__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-BadApples-NightlyTests-master_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001_tempDir-002_jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/c/u, hostPort=35515, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/shard-1-001/cores}
[junit4] 2> 1982581 ERROR (closeThreadPool-4692-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1982581 INFO (closeThreadPool-4692-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1982581 INFO (closeThreadPool-4692-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 1982582 INFO (closeThreadPool-4692-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1982582 INFO (closeThreadPool-4692-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1982582 INFO (closeThreadPool-4692-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-01-19T15:47:35.562Z
[junit4] 2> 1982587 INFO (zkConnectionManagerCallback-4694-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1982588 INFO (closeThreadPool-4692-thread-1) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1982588 INFO (closeThreadPool-4692-thread-1) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/shard-1-001/solr.xml
[junit4] 2> 1982593 INFO (closeThreadPool-4692-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1982593 INFO (closeThreadPool-4692-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1982595 INFO (closeThreadPool-4692-thread-1) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1982807 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/shard-2-001 of type NRT
[junit4] 2> 1982823 WARN (closeThreadPool-4692-thread-2) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 1982823 INFO (closeThreadPool-4692-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 1982823 INFO (closeThreadPool-4692-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 1982823 INFO (closeThreadPool-4692-thread-2) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 1982828 INFO (closeThreadPool-4692-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1982828 INFO (closeThreadPool-4692-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1982828 INFO (closeThreadPool-4692-thread-2) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1982834 INFO (closeThreadPool-4692-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@450cc04f{/c/u,null,AVAILABLE}
[junit4] 2> 1982834 WARN (closeThreadPool-4692-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@f123e41[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1982835 INFO (closeThreadPool-4692-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@67a2bc67{SSL,[ssl, http/1.1]}{127.0.0.1:32823}
[junit4] 2> 1982835 INFO (closeThreadPool-4692-thread-2) [ ] o.e.j.s.Server Started @1982901ms
[junit4] 2> 1982835 INFO (closeThreadPool-4692-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:42794/hdfs__localhost_42794__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-BadApples-NightlyTests-master_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001_tempDir-002_jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/c/u, hostPort=32823, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/shard-2-001/cores}
[junit4] 2> 1982835 ERROR (closeThreadPool-4692-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1982835 INFO (closeThreadPool-4692-thread-2) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1982836 INFO (closeThreadPool-4692-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 1982836 INFO (closeThreadPool-4692-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1982836 INFO (closeThreadPool-4692-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1982836 INFO (closeThreadPool-4692-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-01-19T15:47:35.816Z
[junit4] 2> 1982843 INFO (zkConnectionManagerCallback-4696-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1982845 INFO (closeThreadPool-4692-thread-2) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1982845 INFO (closeThreadPool-4692-thread-2) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/shard-2-001/solr.xml
[junit4] 2> 1982850 INFO (closeThreadPool-4692-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1982850 INFO (closeThreadPool-4692-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1982852 INFO (closeThreadPool-4692-thread-2) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1983016 INFO (closeThreadPool-4692-thread-2) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 1983017 WARN (closeThreadPool-4692-thread-2) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 1983018 WARN (closeThreadPool-4692-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@4d787a4d[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1983022 WARN (closeThreadPool-4692-thread-2) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 1983023 WARN (closeThreadPool-4692-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@7c0ad810[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1983025 INFO (closeThreadPool-4692-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40686/solr
[junit4] 2> 1983026 INFO (zkConnectionManagerCallback-4702-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1983029 INFO (zkConnectionManagerCallback-4704-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1983036 INFO (closeThreadPool-4692-thread-2) [n:127.0.0.1:32823_c%2Fu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1983042 INFO (closeThreadPool-4692-thread-2) [n:127.0.0.1:32823_c%2Fu ] o.a.s.c.ZkController Publish node=127.0.0.1:32823_c%2Fu as DOWN
[junit4] 2> 1983044 INFO (closeThreadPool-4692-thread-2) [n:127.0.0.1:32823_c%2Fu ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1983044 INFO (closeThreadPool-4692-thread-2) [n:127.0.0.1:32823_c%2Fu ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32823_c%2Fu
[junit4] 2> 1983046 INFO (zkCallback-4690-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1983046 INFO (zkCallback-4671-thread-4) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1983046 INFO (zkCallback-4678-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1983047 INFO (zkCallback-4703-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1983067 INFO (closeThreadPool-4692-thread-1) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 1983068 WARN (closeThreadPool-4692-thread-1) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 1983081 WARN (closeThreadPool-4692-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@17261b11[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1983087 WARN (closeThreadPool-4692-thread-1) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 1983088 WARN (closeThreadPool-4692-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@7b99b734[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1983089 INFO (closeThreadPool-4692-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40686/solr
[junit4] 2> 1983093 INFO (zkConnectionManagerCallback-4712-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1983093 INFO (zkConnectionManagerCallback-4717-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1983096 INFO (closeThreadPool-4692-thread-2) [n:127.0.0.1:32823_c%2Fu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1983096 INFO (zkConnectionManagerCallback-4719-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1983102 INFO (closeThreadPool-4692-thread-2) [n:127.0.0.1:32823_c%2Fu ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40686/solr ready
[junit4] 2> 1983103 INFO (closeThreadPool-4692-thread-2) [n:127.0.0.1:32823_c%2Fu ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1983113 INFO (closeThreadPool-4692-thread-1) [n:127.0.0.1:35515_c%2Fu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1983118 INFO (closeThreadPool-4692-thread-1) [n:127.0.0.1:35515_c%2Fu ] o.a.s.c.ZkController Publish node=127.0.0.1:35515_c%2Fu as DOWN
[junit4] 2> 1983119 INFO (closeThreadPool-4692-thread-1) [n:127.0.0.1:35515_c%2Fu ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1983119 INFO (closeThreadPool-4692-thread-1) [n:127.0.0.1:35515_c%2Fu ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35515_c%2Fu
[junit4] 2> 1983121 INFO (zkCallback-4678-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1983121 INFO (zkCallback-4703-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1983121 INFO (zkCallback-4690-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1983121 INFO (zkCallback-4671-thread-4) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1983128 INFO (zkCallback-4718-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1983129 INFO (zkCallback-4716-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1983160 INFO (closeThreadPool-4692-thread-2) [n:127.0.0.1:32823_c%2Fu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea
[junit4] 2> 1983166 INFO (zkConnectionManagerCallback-4727-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1983166 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[1B990FFF714B6410]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/shard-3-001 of type NRT
[junit4] 2> 1983167 INFO (closeThreadPool-4692-thread-1) [n:127.0.0.1:35515_c%2Fu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1983168 WARN (closeThreadPool-4692-thread-3) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 1983170 INFO (closeThreadPool-4692-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 1983170 INFO (closeThreadPool-4692-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 1983170 INFO (closeThreadPool-4692-thread-3) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 1983171 INFO (closeThreadPool-4692-thread-1) [n:127.0.0.1:35515_c%2Fu ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40686/solr ready
[junit4] 2> 1983172 INFO (closeThreadPool-4692-thread-3) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1983172 INFO (closeThreadPool-4692-thread-3) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1983172 INFO (closeThreadPool-4692-thread-3) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1983172 INFO (closeThreadPool-4692-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@64549b03{/c/u,null,AVAILABLE}
[junit4] 2> 1983173 WARN (closeThreadPool-4692-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@1aeb2eb[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1983173 INFO (closeThreadPool-4692-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@10b373c5{SSL,[ssl, http/1.1]}{127.0.0.1:41111}
[junit4] 2> 1983173 INFO (closeThreadPool-4692-thread-3) [ ] o.e.j.s.Server Started @1983240ms
[junit4] 2> 1983173 INFO (closeThreadPool-4692-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:42794/hdfs__localhost_42794__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-BadApples-NightlyTests-master_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001_tempDir-002_jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/c/u, hostPort=41111, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/shard-3-001/cores}
[junit4] 2> 1983173 INFO (closeThreadPool-4692-thread-1) [n:127.0.0.1:35515_c%2Fu ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1983174 ERROR (closeThreadPool-4692-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1983174 INFO (closeThreadPool-4692-thread-3) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1983174 INFO (closeThreadPool-4692-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 9.0.0
[junit4] 2> 1983174 INFO (closeThreadPool-4692-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1983174 INFO (closeThreadPool-4692-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1983174 INFO (closeThreadPool-4692-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-01-19T15:47:36.154Z
[junit4] 2> 1983183 INFO (closeThreadPool-4692-thread-2) [n:127.0.0.1:32823_c%2Fu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea
[junit4] 2> 1983184 INFO (closeThreadPool-4692-thread-2) [n:127.0.0.1:32823_c%2Fu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea
[junit4] 2> 1983186 INFO (closeThreadPool-4692-thread-2) [n:127.0.0.1:32823_c%2Fu ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/shard-2-001/cores
[junit4] 2> 1983197 INFO (zkConnectionManagerCallback-4730-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1983198 INFO (closeThreadPool-4692-thread-3) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1983198 INFO (closeThreadPool-4692-thread-3) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/shard-3-001/solr.xml
[junit4] 2> 1983203 INFO (closeThreadPool-4692-thread-3) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1983203 INFO (closeThreadPool-4692-thread-3) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1983206 INFO (closeThreadPool-4692-thread-3) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1983229 INFO (closeThreadPool-4692-thread-1) [n:127.0.0.1:35515_c%2Fu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea
[junit4] 2> 1983255 INFO (closeThreadPool-4692-thread-1) [n:127.0.0.1:35515_c%2Fu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea
[junit4] 2> 1983255 INFO (closeThreadPool-4692-thread-1) [n:127.0.0.1:35515_c%2Fu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea
[junit4] 2> 1983257 INFO (closeThreadPool-4692-thread-1) [n:127.0.0.1:35515_c%2Fu ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/shard-1-001/cores
[junit4] 2> 1983438 INFO (closeThreadPool-4692-thread-3) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 1983438 WARN (closeThreadPool-4692-thread-3) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 1983440 WARN (closeThreadPool-4692-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@429c4907[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1983443 WARN (closeThreadPool-4692-thread-3) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 1983444 WARN (closeThreadPool-4692-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@47f457c9[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1983445 INFO (closeThreadPool-4692-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40686/solr
[junit4] 2> 1983447 INFO (zkConnectionManagerCallback-4736-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1983450 INFO (zkConnectionManagerCallback-4738-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1983458 INFO (closeThreadPool-4692-thread-3) [n:127.0.0.1:41111_c%2Fu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1983463 INFO (closeThreadPool-4692-thread-3) [n:127.0.0.1:41111_c%2Fu ] o.a.s.c.ZkController Publish node=127.0.0.1:41111_c%2Fu as DOWN
[junit4] 2> 1983465 INFO (closeThreadPool-4692-thread-3) [n:127.0.0.1:41111_c%2Fu ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1983465 INFO (closeThreadPool-4692-thread-3) [n:127.0.0.1:41111_c%2Fu ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41111_c%2Fu
[junit4] 2> 1983467 INFO (zkCallback-4690-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1983467 INFO (zkCallback-4671-thread-4) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1983468 INFO (zkCallback-4703-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1983468 INFO (zkCallback-4678-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1983468 INFO (zkCallback-4718-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1983468 INFO (zkCallback-4716-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1983469 INFO (zkCallback-4737-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1983469 INFO (zkCallback-4726-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1983497 INFO (zkConnectionManagerCallback-4745-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1983500 INFO (closeThreadPool-4692-thread-3) [n:127.0.0.1:41111_c%2Fu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 1983502 INFO (closeThreadPool-4692-thread-3) [n:127.0.0.1:41111_c%2Fu ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40686/solr ready
[junit4] 2> 1983502 INFO (closeThreadPool-4692-thread-3) [n:127.0.0.1:41111_c%2Fu ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1983534 INFO (closeThreadPool-4692-thread-3) [n:127.0.0.1:41111_c%2Fu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea
[junit4] 2> 1983551 INFO (closeThreadPool-4692-thread-3) [n:127.0.0.1:41111_c%2Fu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea
[junit4] 2> 1983551 INFO (closeThreadPool-4692-thread-3) [n:127.0.0.1:41111_c%2Fu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea
[junit4] 2> 1983553 INFO (closeThreadPool-4692-thread-3) [n:127.0.0.1:41111_c%2Fu ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/shard-3-001/cores
[junit4] 2> 1983653 INFO (qtp964604967-63492) [n:127.0.0.1:32823_c%2Fu ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:35515_c%252Fu&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1983670 INFO (qtp964604967-63493) [n:127.0.0.1:32823_c%2Fu ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:32823_c%252Fu&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1983671 INFO (OverseerCollectionConfigSetProcessor-72650538903797765-127.0.0.1:36572_c%2Fu-n_0000000000) [n:127.0.0.1:36572_c%2Fu ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1983678 INFO (qtp964604967-63491) [n:127.0.0.1:32823_c%2Fu ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41111_c%252Fu&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1983682 INFO (qtp964604967-63490) [n:127.0.0.1:32823_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983683 INFO (qtp964604967-63489) [n:127.0.0.1:32823_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983684 INFO (qtp964604967-63490) [n:127.0.0.1:32823_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983685 INFO (qtp964604967-63489) [n:127.0.0.1:32823_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983686 INFO (qtp964604967-63490) [n:127.0.0.1:32823_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983712 INFO (qtp550116991-63478) [n:127.0.0.1:35515_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983714 INFO (qtp550116991-63479) [n:127.0.0.1:35515_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983715 INFO (qtp550116991-63480) [n:127.0.0.1:35515_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983716 INFO (qtp550116991-63474) [n:127.0.0.1:35515_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983717 INFO (qtp550116991-63478) [n:127.0.0.1:35515_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983720 INFO (qtp495006951-63413) [n:127.0.0.1:36572_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 1983722 INFO (qtp495006951-63414) [n:127.0.0.1:36572_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 1983723 INFO (qtp495006951-63415) [n:127.0.0.1:36572_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 1983724 INFO (qtp495006951-63412) [n:127.0.0.1:36572_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983725 INFO (qtp495006951-63416) [n:127.0.0.1:36572_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983726 INFO (qtp495006951-63413) [n:127.0.0.1:36572_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983727 INFO (qtp495006951-63414) [n:127.0.0.1:36572_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983728 INFO (qtp495006951-63415) [n:127.0.0.1:36572_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983745 INFO (qtp1036819764-63537) [n:127.0.0.1:41111_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983747 INFO (qtp1036819764-63538) [n:127.0.0.1:41111_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983748 INFO (qtp1036819764-63539) [n:127.0.0.1:41111_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983749 INFO (qtp1036819764-63540) [n:127.0.0.1:41111_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983750 INFO (qtp1036819764-63542) [n:127.0.0.1:41111_c%2Fu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1983752 INFO (OverseerThreadFactory-4786-thread-3-processing-n:127.0.0.1:36572_c%2Fu) [n:127.0.0.1:36572_c%2Fu c:collection1 s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:35515_c%2Fu for creating new replica of shard shard1 for collection collection1
[junit4] 2> 1983761 INFO (OverseerThreadFactory-4786-thread-3-processing-n:127.0.0.1:36572_c%2Fu) [n:127.0.0.1:36572_c%2Fu c:collection1 s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 1983791 INFO (qtp550116991-63479) [n:127.0.0.1:35515_c%2Fu x:collection1_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n1&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1984828 INFO (qtp550116991-63479) [n:127.0.0.1:35515_c%2Fu c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 1984857 INFO (qtp550116991-63479) [n:127.0.0.1:35515_c%2Fu c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema [collection1_shard1_replica_n1] Schema name=test
[junit4] 2> 1984994 INFO (qtp550116991-63479) [n:127.0.0.1:35515_c%2Fu c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1985022 INFO (qtp550116991-63479) [n:127.0.0.1:35515_c%2Fu c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n1' using configuration from collection collection1, trusted=true
[junit4] 2> 1985023 INFO (qtp550116991-63479) [n:127.0.0.1:35515_c%2Fu c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n1' (registry 'solr.core.collection1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c8a08ea
[junit4] 2> 1985023 INFO (qtp550116991-63479) [n:127.0.0.1:35515_c%2Fu c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [[collection1_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/shard-1-001/cores/collection1_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_1B990FFF714B6410-001/shard-1-001/cores/collection1_shard1_replica_n1/data/]
[junit4] 2> 1985027 INFO (qtp550116991-63479) [n:127.0.0.1:35515_c%2Fu c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=97.541015625, floorSegmentMB=1.357421875, forceMergeDeletesPctAllowed=11.115188937090268, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5987367564478258, deletesPctAllowed=33.419722422116344
[junit4] 2> 1985038 WARN (qtp550116991-63479) [n:127.0.0.1:35515_c%2Fu c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1985128 INFO (qtp550116991-63479) [n:127.0.0.1:35515_c%2Fu c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1985128 INFO (qtp550116991-63479) [n:127.0.0.1:35515_c%2Fu c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1985131 INFO (qtp550116991-63479) [n:127.0.0.1:35515_c%2Fu c:collection1 s:shard1 x:collection1_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for
[...truncated too long message...]
-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/bcprov-jdk15on-1.60.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/byte-buddy-1.9.3.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/mockito-core-2.23.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/test-lib/objenesis-2.6.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-9.0.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/build/backward-codecs/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/contrib/analysis-extras/lib/icu4j-62.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.4.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_191/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.7.2.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20190119_151432_8288247385733212930642.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20190119_151432_8281767076447586153977.suites -stdin
[junit4] ERROR: JVM J2 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1563)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:123)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:997)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:994)
[junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] at java.lang.Thread.run(Thread.java:748)
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/build.xml:661: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/build.xml:577: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build.xml:267: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/common-build.xml:558: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/common-build.xml:1572: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/common-build.xml:1099: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
Total time: 307 minutes 54 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-BadApples-NightlyTests-master #43 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-BadApples-NightlyTests-master - Build # 45 -
Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-master/45/
4 tests failed.
FAILED: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test
Error Message:
Captured an uncaught exception in thread: Thread[id=7696, name=Thread-2047, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=7696, name=Thread-2047, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
at __randomizedtesting.SeedInfo.seed([CE7B6F721C0D523A:462F50A8B2F13FC2]:0)
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:35806//control_collection
at __randomizedtesting.SeedInfo.seed([CE7B6F721C0D523A]:0)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:638)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:35806//control_collection
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:661)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:256)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:245)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:224)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:636)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:549)
... 5 more
FAILED: org.apache.solr.cloud.RestartWhileUpdatingTest.test
Error Message:
There are still nodes recoverying - waited for 320 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
at __randomizedtesting.SeedInfo.seed([CE7B6F721C0D523A:462F50A8B2F13FC2]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:195)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1038)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1595)
at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:144)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1070)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1042)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.update.RootFieldTest
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
FAILED: org.apache.solr.uninverting.TestDocTermOrdsUninvertLimit.testTriggerUnInvertLimit
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
at org.apache.lucene.util.ByteBlockPool$DirectTrackingAllocator.getByteBlock(ByteBlockPool.java:103)
at org.apache.lucene.util.ByteBlockPool.nextBuffer(ByteBlockPool.java:203)
at org.apache.lucene.util.ByteBlockPool.allocSlice(ByteBlockPool.java:258)
at org.apache.lucene.index.TermsHashPerField.writeByte(TermsHashPerField.java:203)
at org.apache.lucene.index.TermsHashPerField.writeBytes(TermsHashPerField.java:215)
at org.apache.lucene.index.FreqProxTermsWriterPerField.writeProx(FreqProxTermsWriterPerField.java:86)
at org.apache.lucene.index.FreqProxTermsWriterPerField.addTerm(FreqProxTermsWriterPerField.java:171)
at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:185)
at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:861)
at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:441)
at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:405)
at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:251)
at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:494)
at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1594)
at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:189)
at org.apache.solr.uninverting.TestDocTermOrdsUninvertLimit.testTriggerUnInvertLimit(TestDocTermOrdsUninvertLimit.java:65)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
Build Log:
[...truncated 14060 lines...]
[junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/init-core-data-001
[junit4] 2> 625112 WARN (SUITE-FullSolrCloudDistribCmdsTest-seed#[CE7B6F721C0D523A]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=24 numCloses=24
[junit4] 2> 625112 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[CE7B6F721C0D523A]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 625114 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[CE7B6F721C0D523A]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 625114 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[CE7B6F721C0D523A]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 625114 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[CE7B6F721C0D523A]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 625122 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 625122 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 625122 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 625222 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.ZkTestServer start zk server on port:40320
[junit4] 2> 625222 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:40320
[junit4] 2> 625222 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 40320
[junit4] 2> 625235 INFO (zkConnectionManagerCallback-1593-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 625242 INFO (zkConnectionManagerCallback-1595-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 625244 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 625245 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 625247 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 625248 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 625249 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 625250 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 625252 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 625253 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 625254 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 625255 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 625257 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 625258 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 625431 WARN (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 625431 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 625431 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 625431 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 625432 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 625432 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 625432 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 625433 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@385e0fcb{/,null,AVAILABLE}
[junit4] 2> 625439 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@50e9b1ec{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:35806}
[junit4] 2> 625439 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.e.j.s.Server Started @625523ms
[junit4] 2> 625439 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/tempDir-001/control/data, hostContext=/, hostPort=35806, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/control-001/cores}
[junit4] 2> 625440 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 625440 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 625440 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 625440 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 625440 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 625440 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-01-11T19:22:43.284Z
[junit4] 2> 625448 INFO (zkConnectionManagerCallback-1597-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 625449 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 625450 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/control-001/solr.xml
[junit4] 2> 625455 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 625455 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 625457 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 625793 WARN (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@4060a13f[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 625801 WARN (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@327d6b05[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 625802 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40320/solr
[junit4] 2> 625804 INFO (zkConnectionManagerCallback-1603-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 625807 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x101f2d9edb10003, likely client has closed socket
[junit4] 2> 625808 INFO (zkConnectionManagerCallback-1605-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 625914 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [n:127.0.0.1:35806_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35806_
[junit4] 2> 625917 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [n:127.0.0.1:35806_ ] o.a.s.c.Overseer Overseer (id=72606086824263684-127.0.0.1:35806_-n_0000000000) starting
[junit4] 2> 625932 INFO (zkConnectionManagerCallback-1612-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 625936 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [n:127.0.0.1:35806_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40320/solr ready
[junit4] 2> 625937 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [n:127.0.0.1:35806_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35806_
[junit4] 2> 625937 INFO (OverseerStateUpdate-72606086824263684-127.0.0.1:35806_-n_0000000000) [n:127.0.0.1:35806_ ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:35806_
[junit4] 2> 625950 INFO (zkCallback-1611-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 625958 INFO (OverseerStateUpdate-72606086824263684-127.0.0.1:35806_-n_0000000000) [n:127.0.0.1:35806_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 625977 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [n:127.0.0.1:35806_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 626003 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [n:127.0.0.1:35806_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6
[junit4] 2> 626018 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [n:127.0.0.1:35806_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6
[junit4] 2> 626019 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [n:127.0.0.1:35806_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6
[junit4] 2> 626020 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [n:127.0.0.1:35806_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/control-001/cores
[junit4] 2> 626062 INFO (zkConnectionManagerCallback-1618-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 626063 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 626068 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40320/solr ready
[junit4] 2> 626070 INFO (qtp1668363493-6991) [n:127.0.0.1:35806_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:35806_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 626077 INFO (OverseerThreadFactory-1562-thread-1-processing-n:127.0.0.1:35806_) [n:127.0.0.1:35806_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 626187 INFO (qtp1668363493-6993) [n:127.0.0.1:35806_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 626188 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 626189 INFO (qtp1668363493-6990) [n:127.0.0.1:35806_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 626190 INFO (qtp1668363493-6992) [n:127.0.0.1:35806_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 626191 INFO (qtp1668363493-6993) [n:127.0.0.1:35806_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 626196 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 626196 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 627225 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 627242 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 627356 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 627376 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 627376 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6
[junit4] 2> 627377 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 627380 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=67.19921875, floorSegmentMB=0.96484375, forceMergeDeletesPctAllowed=19.808106997165734, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=46.33019046355927
[junit4] 2> 627389 WARN (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 627498 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 627498 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 627501 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 627501 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 627503 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@67c193a0
[junit4] 2> 627503 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3afd012c[control_collection_shard1_replica_n1] main]
[junit4] 2> 627505 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 627508 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 627508 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 627509 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1622393031599587328
[junit4] 2> 627527 INFO (searcherExecutor-1567-thread-1-processing-n:127.0.0.1:35806_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@3afd012c[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 627542 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 627542 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
[junit4] 2> 627546 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 627546 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 627546 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:35806/control_collection_shard1_replica_n1/
[junit4] 2> 627546 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 627547 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:35806/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 627547 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72606086824263684-core_node2-n_0000000000
[junit4] 2> 627549 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:35806/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 627651 INFO (zkCallback-1604-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 627652 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 627665 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1469
[junit4] 2> 627669 INFO (zkCallback-1604-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 627676 INFO (qtp1668363493-6991) [n:127.0.0.1:35806_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 627774 INFO (zkCallback-1604-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 627774 INFO (zkCallback-1604-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 627775 INFO (qtp1668363493-6991) [n:127.0.0.1:35806_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:35806_&wt=javabin&version=2} status=0 QTime=1704
[junit4] 2> 627779 INFO (zkCallback-1604-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 627811 INFO (zkConnectionManagerCallback-1623-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 627812 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 627814 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40320/solr ready
[junit4] 2> 627814 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 627815 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 627822 INFO (OverseerCollectionConfigSetProcessor-72606086824263684-127.0.0.1:35806_-n_0000000000) [n:127.0.0.1:35806_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 627827 INFO (OverseerThreadFactory-1562-thread-2-processing-n:127.0.0.1:35806_) [n:127.0.0.1:35806_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 628036 WARN (OverseerThreadFactory-1562-thread-2-processing-n:127.0.0.1:35806_) [n:127.0.0.1:35806_ ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 628052 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 628054 INFO (qtp1668363493-6994) [n:127.0.0.1:35806_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=238
[junit4] 2> 628058 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=6
[junit4] 2> 628321 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-1-001 of type NRT
[junit4] 2> 628336 WARN (closeThreadPool-1624-thread-1) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 628336 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 628336 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 628336 INFO (closeThreadPool-1624-thread-1) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 628366 INFO (closeThreadPool-1624-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 628366 INFO (closeThreadPool-1624-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 628366 INFO (closeThreadPool-1624-thread-1) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 628366 INFO (closeThreadPool-1624-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3ffa6dc7{/,null,AVAILABLE}
[junit4] 2> 628367 INFO (closeThreadPool-1624-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@a319207{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:43692}
[junit4] 2> 628367 INFO (closeThreadPool-1624-thread-1) [ ] o.e.j.s.Server Started @628451ms
[junit4] 2> 628367 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=43692, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-1-001/cores}
[junit4] 2> 628367 ERROR (closeThreadPool-1624-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 628367 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 628368 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 628368 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 628368 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 628368 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-01-11T19:22:46.212Z
[junit4] 2> 628391 INFO (zkConnectionManagerCallback-1626-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 628392 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 628392 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-1-001/solr.xml
[junit4] 2> 628398 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 628399 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 628400 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 628617 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-2-001 of type NRT
[junit4] 2> 628626 WARN (closeThreadPool-1624-thread-2) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 628626 INFO (closeThreadPool-1624-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 628626 INFO (closeThreadPool-1624-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 628626 INFO (closeThreadPool-1624-thread-2) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 628667 INFO (closeThreadPool-1624-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 628667 INFO (closeThreadPool-1624-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 628667 INFO (closeThreadPool-1624-thread-2) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 628675 INFO (closeThreadPool-1624-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3a7ff71d{/,null,AVAILABLE}
[junit4] 2> 628676 INFO (closeThreadPool-1624-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@448b5cdc{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:40212}
[junit4] 2> 628676 INFO (closeThreadPool-1624-thread-2) [ ] o.e.j.s.Server Started @628760ms
[junit4] 2> 628676 INFO (closeThreadPool-1624-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=40212, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-2-001/cores}
[junit4] 2> 628677 ERROR (closeThreadPool-1624-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 628677 INFO (closeThreadPool-1624-thread-2) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 628677 INFO (closeThreadPool-1624-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 628677 INFO (closeThreadPool-1624-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 628677 INFO (closeThreadPool-1624-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 628677 INFO (closeThreadPool-1624-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-01-11T19:22:46.521Z
[junit4] 2> 628695 INFO (zkConnectionManagerCallback-1628-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 628696 INFO (closeThreadPool-1624-thread-2) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 628696 INFO (closeThreadPool-1624-thread-2) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-2-001/solr.xml
[junit4] 2> 628703 INFO (closeThreadPool-1624-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 628703 INFO (closeThreadPool-1624-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 628705 INFO (closeThreadPool-1624-thread-2) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 628822 WARN (closeThreadPool-1624-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@612c06ee[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 628828 WARN (closeThreadPool-1624-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@12d56eae[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 628829 INFO (closeThreadPool-1624-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40320/solr
[junit4] 2> 628831 INFO (zkConnectionManagerCallback-1634-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 628834 INFO (zkConnectionManagerCallback-1636-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 628840 INFO (closeThreadPool-1624-thread-2) [n:127.0.0.1:40212_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 628844 WARN (closeThreadPool-1624-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@311b52d[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 628845 INFO (closeThreadPool-1624-thread-2) [n:127.0.0.1:40212_ ] o.a.s.c.ZkController Publish node=127.0.0.1:40212_ as DOWN
[junit4] 2> 628846 INFO (closeThreadPool-1624-thread-2) [n:127.0.0.1:40212_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 628846 INFO (closeThreadPool-1624-thread-2) [n:127.0.0.1:40212_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40212_
[junit4] 2> 628849 INFO (zkCallback-1604-thread-3) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 628849 WARN (closeThreadPool-1624-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@3d843b79[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 628849 INFO (zkCallback-1611-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 628850 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40320/solr
[junit4] 2> 628851 INFO (zkCallback-1622-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 628853 INFO (zkConnectionManagerCallback-1644-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 628856 INFO (zkConnectionManagerCallback-1646-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 628861 INFO (zkCallback-1635-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 628862 INFO (closeThreadPool-1624-thread-1) [n:127.0.0.1:43692_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 628867 INFO (closeThreadPool-1624-thread-1) [n:127.0.0.1:43692_ ] o.a.s.c.ZkController Publish node=127.0.0.1:43692_ as DOWN
[junit4] 2> 628868 INFO (closeThreadPool-1624-thread-1) [n:127.0.0.1:43692_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 628868 INFO (closeThreadPool-1624-thread-1) [n:127.0.0.1:43692_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43692_
[junit4] 2> 628870 INFO (zkCallback-1622-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 628870 INFO (zkCallback-1611-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 628870 INFO (zkCallback-1635-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 628870 INFO (zkCallback-1604-thread-3) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 628887 INFO (zkCallback-1645-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 628900 INFO (zkConnectionManagerCallback-1653-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 628903 INFO (closeThreadPool-1624-thread-1) [n:127.0.0.1:43692_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 628905 INFO (closeThreadPool-1624-thread-1) [n:127.0.0.1:43692_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40320/solr ready
[junit4] 2> 628911 INFO (closeThreadPool-1624-thread-1) [n:127.0.0.1:43692_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 628916 INFO (zkConnectionManagerCallback-1659-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 628918 INFO (closeThreadPool-1624-thread-2) [n:127.0.0.1:40212_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 628923 INFO (closeThreadPool-1624-thread-2) [n:127.0.0.1:40212_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40320/solr ready
[junit4] 2> 628927 INFO (closeThreadPool-1624-thread-2) [n:127.0.0.1:40212_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 628990 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-3-001 of type NRT
[junit4] 2> 628999 WARN (closeThreadPool-1624-thread-3) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 629000 INFO (closeThreadPool-1624-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 629000 INFO (closeThreadPool-1624-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 629000 INFO (closeThreadPool-1624-thread-3) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 629043 WARN (closeThreadPool-1624-thread-2) [n:127.0.0.1:40212_ ] o.a.s.m.r.j.JmxMetricsReporter Unable to register timer
[junit4] 2> javax.management.InstanceNotFoundException: solr:dom1=node,category=ADMIN,scope=/admin/configs,name=requestTimes
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1095) ~[?:1.8.0_191]
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:427) ~[?:1.8.0_191]
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415) ~[?:1.8.0_191]
[junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546) ~[?:1.8.0_191]
[junit4] 2> at org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.registerMBean(JmxMetricsReporter.java:531) ~[java/:?]
[junit4] 2> at org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.onTimerAdded(JmxMetricsReporter.java:674) ~[java/:?]
[junit4] 2> at org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.lambda$start$0(JmxMetricsReporter.java:740) ~[java/:?]
[junit4] 2> at java.util.HashMap.forEach(HashMap.java:1289) ~[?:1.8.0_191]
[junit4] 2> at org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.start(JmxMetricsReporter.java:732) ~[java/:?]
[junit4] 2> at org.apache.solr.metrics.reporters.SolrJmxReporter.doInit(SolrJmxReporter.java:110) ~[java/:?]
[junit4] 2> at org.apache.solr.metrics.SolrMetricReporter.init(SolrMetricReporter.java:70) ~[java/:?]
[junit4] 2> at org.apache.solr.metrics.SolrMetricManager.loadReporter(SolrMetricManager.java:916) ~[java/:?]
[junit4] 2> at org.apache.solr.metrics.SolrMetricManager.loadReporters(SolrMetricManager.java:843) ~[java/:?]
[junit4] 2> at org.apache.solr.core.CoreContainer.load(CoreContainer.java:600) ~[java/:?]
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:252) ~[java/:?]
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:172) ~[java/:?]
[junit4] 2> at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:136) ~[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.lambda$initialize$0(ServletHandler.java:750) ~[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948) ~[?:1.8.0_191]
[junit4] 2> at java.util.stream.Streams$ConcatSpliterator.forEachRemaining(Streams.java:742) ~[?:1.8.0_191]
[junit4] 2> at java.util.stream.Streams$ConcatSpliterator.forEachRemaining(Streams.java:742) ~[?:1.8.0_191]
[junit4] 2> at java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:580) ~[?:1.8.0_191]
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:744) ~[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1449) ~[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1513) ~[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1158) ~[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:995) ~[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:467) ~[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:381) ~[java/:?]
[junit4] 2> at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) ~[jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) ~[jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:544) ~[java/:?]
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:483) ~[java/:?]
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:451) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.AbstractFullDistribZkTestBase.lambda$createJettys$2(AbstractFullDistribZkTestBase.java:464) ~[java/:?]
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_191]
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_191]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_191]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_191]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
[junit4] 2> 629071 INFO (closeThreadPool-1624-thread-3) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 629071 INFO (closeThreadPool-1624-thread-3) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 629071 INFO (closeThreadPool-1624-thread-3) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 629075 INFO (closeThreadPool-1624-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@50ce7393{/,null,AVAILABLE}
[junit4] 2> 629078 INFO (closeThreadPool-1624-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2c5f53cc{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:39382}
[junit4] 2> 629078 INFO (closeThreadPool-1624-thread-3) [ ] o.e.j.s.Server Started @629162ms
[junit4] 2> 629078 INFO (closeThreadPool-1624-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=39382, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-3-001/cores}
[junit4] 2> 629079 ERROR (closeThreadPool-1624-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 629079 INFO (closeThreadPool-1624-thread-3) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 629079 INFO (closeThreadPool-1624-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 629079 INFO (closeThreadPool-1624-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 629079 INFO (closeThreadPool-1624-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 629079 INFO (closeThreadPool-1624-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-01-11T19:22:46.923Z
[junit4] 2> 629080 INFO (closeThreadPool-1624-thread-2) [n:127.0.0.1:40212_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6
[junit4] 2> 629096 INFO (closeThreadPool-1624-thread-1) [n:127.0.0.1:43692_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6
[junit4] 2> 629126 INFO (closeThreadPool-1624-thread-2) [n:127.0.0.1:40212_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6
[junit4] 2> 629126 INFO (closeThreadPool-1624-thread-2) [n:127.0.0.1:40212_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6
[junit4] 2> 629161 INFO (closeThreadPool-1624-thread-2) [n:127.0.0.1:40212_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-2-001/cores
[junit4] 2> 629172 INFO (zkConnectionManagerCallback-1662-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 629183 INFO (closeThreadPool-1624-thread-3) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 629183 INFO (closeThreadPool-1624-thread-3) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-3-001/solr.xml
[junit4] 2> 629191 INFO (closeThreadPool-1624-thread-3) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 629191 INFO (closeThreadPool-1624-thread-3) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 629193 INFO (closeThreadPool-1624-thread-3) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 629209 INFO (closeThreadPool-1624-thread-1) [n:127.0.0.1:43692_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6
[junit4] 2> 629210 INFO (closeThreadPool-1624-thread-1) [n:127.0.0.1:43692_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6
[junit4] 2> 629212 INFO (closeThreadPool-1624-thread-1) [n:127.0.0.1:43692_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-1-001/cores
[junit4] 2> 629335 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-4-001 of type NRT
[junit4] 2> 629360 WARN (closeThreadPool-1624-thread-4) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 629361 INFO (closeThreadPool-1624-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 629361 INFO (closeThreadPool-1624-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 629361 INFO (closeThreadPool-1624-thread-4) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 629409 INFO (closeThreadPool-1624-thread-4) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 629409 INFO (closeThreadPool-1624-thread-4) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 629409 INFO (closeThreadPool-1624-thread-4) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 629410 INFO (closeThreadPool-1624-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@18854f42{/,null,AVAILABLE}
[junit4] 2> 629411 INFO (closeThreadPool-1624-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1f3db0f3{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:43816}
[junit4] 2> 629411 INFO (closeThreadPool-1624-thread-4) [ ] o.e.j.s.Server Started @629494ms
[junit4] 2> 629411 INFO (closeThreadPool-1624-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/tempDir-001/jetty4, solrconfig=solrconfig.xml, hostContext=/, hostPort=43816, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-4-001/cores}
[junit4] 2> 629411 ERROR (closeThreadPool-1624-thread-4) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 629411 INFO (closeThreadPool-1624-thread-4) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 629411 INFO (closeThreadPool-1624-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 629411 INFO (closeThreadPool-1624-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 629411 INFO (closeThreadPool-1624-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 629411 INFO (closeThreadPool-1624-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-01-11T19:22:47.255Z
[junit4] 2> 629455 INFO (zkConnectionManagerCallback-1664-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 629456 INFO (closeThreadPool-1624-thread-4) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 629456 INFO (closeThreadPool-1624-thread-4) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-4-001/solr.xml
[junit4] 2> 629461 INFO (closeThreadPool-1624-thread-4) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 629461 INFO (closeThreadPool-1624-thread-4) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 629470 INFO (closeThreadPool-1624-thread-4) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 629652 WARN (closeThreadPool-1624-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@370141b5[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 629660 WARN (closeThreadPool-1624-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@719c45ba[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 629661 INFO (closeThreadPool-1624-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40320/solr
[junit4] 2> 629687 INFO (zkConnectionManagerCallback-1670-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 629691 INFO (zkConnectionManagerCallback-1672-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 629692 WARN (closeThreadPool-1624-thread-4) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@54d949b9[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 629700 WARN (closeThreadPool-1624-thread-4) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@12574fc9[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 629701 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 5 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-5-001 of type NRT
[junit4] 2> 629701 INFO (closeThreadPool-1624-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40320/solr
[junit4] 2> 629701 WARN (closeThreadPool-1624-thread-1) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 629702 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 629702 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 629702 INFO (closeThreadPool-1624-thread-1) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 629706 INFO (closeThreadPool-1624-thread-3) [n:127.0.0.1:39382_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 629711 INFO (closeThreadPool-1624-thread-3) [n:127.0.0.1:39382_ ] o.a.s.c.ZkController Publish node=127.0.0.1:39382_ as DOWN
[junit4] 2> 629712 INFO (closeThreadPool-1624-thread-3) [n:127.0.0.1:39382_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 629712 INFO (closeThreadPool-1624-thread-3) [n:127.0.0.1:39382_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39382_
[junit4] 2> 629714 INFO (zkCallback-1635-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 629714 INFO (zkCallback-1611-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 629714 INFO (zkCallback-1622-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 629714 INFO (zkCallback-1604-thread-3) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 629723 INFO (zkCallback-1645-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 629724 INFO (zkCallback-1652-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 629729 INFO (closeThreadPool-1624-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 629729 INFO (closeThreadPool-1624-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 629730 INFO (closeThreadPool-1624-thread-1) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 629730 INFO (closeThreadPool-1624-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@58540ff7{/,null,AVAILABLE}
[junit4] 2> 629736 INFO (zkConnectionManagerCallback-1678-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 629749 INFO (closeThreadPool-1624-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5a309bc5{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:41684}
[junit4] 2> 629750 INFO (closeThreadPool-1624-thread-1) [ ] o.e.j.s.Server Started @629833ms
[junit4] 2> 629750 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/tempDir-001/jetty5, solrconfig=solrconfig.xml, hostContext=/, hostPort=41684, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-5-001/cores}
[junit4] 2> 629785 ERROR (closeThreadPool-1624-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 629785 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 629785 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 629785 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 629785 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 629785 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2019-01-11T19:22:47.629Z
[junit4] 2> 629787 INFO (zkCallback-1658-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 629787 INFO (zkCallback-1671-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 629806 INFO (zkConnectionManagerCallback-1687-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 629806 INFO (zkConnectionManagerCallback-1685-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 629806 INFO (zkConnectionManagerCallback-1689-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 629809 INFO (closeThreadPool-1624-thread-3) [n:127.0.0.1:39382_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 629812 INFO (closeThreadPool-1624-thread-3) [n:127.0.0.1:39382_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40320/solr ready
[junit4] 2> 629815 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x101f2d9edb10017, likely client has closed socket
[junit4] 2> 629823 INFO (closeThreadPool-1624-thread-3) [n:127.0.0.1:39382_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 629836 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 629836 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-5-001/solr.xml
[junit4] 2> 629846 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 629846 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 629856 INFO (closeThreadPool-1624-thread-1) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 629866 INFO (closeThreadPool-1624-thread-4) [n:127.0.0.1:43816_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 629867 INFO (OverseerCollectionConfigSetProcessor-72606086824263684-127.0.0.1:35806_-n_0000000000) [n:127.0.0.1:35806_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 629873 INFO (closeThreadPool-1624-thread-4) [n:127.0.0.1:43816_ ] o.a.s.c.ZkController Publish node=127.0.0.1:43816_ as DOWN
[junit4] 2> 629874 INFO (closeThreadPool-1624-thread-4) [n:127.0.0.1:43816_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 629874 INFO (closeThreadPool-1624-thread-4) [n:127.0.0.1:43816_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43816_
[junit4] 2> 629877 INFO (zkCallback-1671-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 629877 INFO (zkCallback-1611-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 629877 INFO (zkCallback-1622-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 629877 INFO (zkCallback-1635-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 629877 INFO (zkCallback-1652-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 629878 INFO (zkCallback-1604-thread-3) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 629878 INFO (zkCallback-1645-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 629899 INFO (zkCallback-1658-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 629899 INFO (zkCallback-1684-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 629905 INFO (zkCallback-1688-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 629912 INFO (closeThreadPool-1624-thread-3) [n:127.0.0.1:39382_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6
[junit4] 2> 629953 INFO (closeThreadPool-1624-thread-3) [n:127.0.0.1:39382_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6
[junit4] 2> 629953 INFO (closeThreadPool-1624-thread-3) [n:127.0.0.1:39382_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6
[junit4] 2> 629955 INFO (zkConnectionManagerCallback-1697-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 629955 INFO (closeThreadPool-1624-thread-3) [n:127.0.0.1:39382_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-3-001/cores
[junit4] 2> 629974 INFO (closeThreadPool-1624-thread-4) [n:127.0.0.1:43816_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
[junit4] 2> 629976 INFO (closeThreadPool-1624-thread-4) [n:127.0.0.1:43816_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40320/solr ready
[junit4] 2> 630001 INFO (closeThreadPool-1624-thread-4) [n:127.0.0.1:43816_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 630062 INFO (closeThreadPool-1624-thread-4) [n:127.0.0.1:43816_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6
[junit4] 2> 630075 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[CE7B6F721C0D523A]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 6 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-6-001 of type NRT
[junit4] 2> 630098 INFO (closeThreadPool-1624-thread-4) [n:127.0.0.1:43816_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6
[junit4] 2> 630099 INFO (closeThreadPool-1624-thread-4) [n:127.0.0.1:43816_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@241c69c6
[junit4] 2> 630101 INFO (closeThreadPool-1624-thread-4) [n:127.0.0.1:43816_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_CE7B6F721C0D523A-001/shard-4-001/cores
[junit4] 2> 630128 WARN (closeThreadPool-1624-thread-2) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 630139 INFO (closeThreadPool-1624-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 630139 INFO (closeThreadPool-1624-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 630139 INFO (closeThreadPool-1624-thread-2) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 630148 INFO (closeThreadPool-1624-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 630148 INFO (closeThreadPool-1624-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 630148 INFO (closeThreadPool-1624-thread-2) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 630149 INFO (closeThreadPool-1624-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@bbf824a{/,null,AVAILABLE}
[junit4] 2> 630154 INFO (closeThreadPool-1624-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4663f5b9{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:39084}
[junit4] 2> 630154 INFO (closeThreadPool-1624-thread-2) [ ] o.e.j.s.Server Started @630238ms
[junit4] 2> 630154 INFO (closeThreadPool-1624-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1
[...truncated too long message...]
nalyzedSuggestions, TestPhraseSuggestions, BlockDirectoryTest, HdfsLockFactoryTest, TestDocTermOrds, TestDocTermOrdsUninvertLimit]
[junit4] Completed [800/837 (3!)] on J1 in 39.32s, 1 test, 1 error <<< FAILURES!
[...truncated 64 lines...]
[junit4] Suite: org.apache.solr.update.RootFieldTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.RootFieldTest_CE7B6F721C0D523A-001/init-core-data-001
[junit4] 2> 5530021 INFO (SUITE-RootFieldTest-seed#[CE7B6F721C0D523A]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 5530024 INFO (SUITE-RootFieldTest-seed#[CE7B6F721C0D523A]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 5530025 INFO (SUITE-RootFieldTest-seed#[CE7B6F721C0D523A]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 5530026 INFO (SUITE-RootFieldTest-seed#[CE7B6F721C0D523A]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore
[junit4] 2> 5530335 INFO (SUITE-RootFieldTest-seed#[CE7B6F721C0D523A]-worker) [ ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 5538280 INFO (SUITE-RootFieldTest-seed#[CE7B6F721C0D523A]-worker) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 5578124 INFO (SUITE-RootFieldTest-seed#[CE7B6F721C0D523A]-worker) [ ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@2c4f5976), locale=ko, timezone=Africa/Dakar
[junit4] 2> NOTE: Linux 4.4.0-137-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=54804616,total=472907776
[junit4] 2> NOTE: All tests run in this JVM: [TestBinaryResponseWriter, SignatureUpdateProcessorFactoryTest, TestMissingGroups, ShufflingReplicaListTransformerTest, TestStressCloudBlindAtomicUpdates, TestChildDocTransformer, LeaderElectionTest, TestSlowCompositeReaderWrapper, TestBadConfig, XmlUpdateRequestHandlerTest, ReplaceNodeNoTargetTest, TestExtendedDismaxParser, SolrIndexConfigTest, AliasIntegrationTest, ConfigureRecoveryStrategyTest, TestTrackingShardHandlerFactory, PeerSyncWithIndexFingerprintCachingTest, TestSubQueryTransformerCrossCore, CoreAdminOperationTest, SyncSliceTest, TestLMDirichletSimilarityFactory, TestMinMaxOnMultiValuedField, EmbeddedSolrNoSerializeTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, OrderedExecutorTest, CurrencyFieldTypeTest, ReplicaListTransformerTest, TestHdfsUpdateLog, TestCloudPivotFacet, MetricsConfigTest, TestRTGBase, TestSmileRequest, AutoscalingHistoryHandlerTest, VMParamsZkACLAndCredentialsProvidersTest, TestLRUStatsCache, TestMacros, ZkStateWriterTest, TestUtils, TestSQLHandlerNonCloud, TestShardHandlerFactory, SaslZkACLProviderTest, DistributedFacetPivotLongTailTest, TestRawTransformer, TestCorePropertiesReload, TestCustomSort, TestSimPolicyCloud, ZkFailoverTest, AddBlockUpdateTest, FullSolrCloudDistribCmdsTest, SolrMetricReporterTest, TestCloudNestedDocsSort, TestManagedSchemaThreadSafety, TestSolrCloudWithDelegationTokens, CircularListTest, SharedFSAutoReplicaFailoverTest, CdcrWithNodesRestartsTest, TestUseDocValuesAsStored, TestFieldCacheVsDocValues, NumericFieldsTest, TestJsonFacetRefinement, TestCryptoKeys, TestSimComputePlanAction, TestRandomFaceting, ConcurrentCreateRoutedAliasTest, DisMaxRequestHandlerTest, TransactionLogTest, CheckHdfsIndexTest, DistributedQueryComponentOptimizationTest, TestDistribIDF, NodeLostTriggerTest, TestImpersonationWithHadoopAuth, WordBreakSolrSpellCheckerTest, PreAnalyzedFieldManagedSchemaCloudTest, ShowFileRequestHandlerTest, VersionInfoTest, HttpPartitionTest, TestCloudRecovery2, TestBinaryField, TestHdfsBackupRestoreCore, TestLazyCores, TestCursorMarkWithoutUniqueKey, TestComplexPhraseLeadingWildcard, DistributedSuggestComponentTest, TestStressUserVersions, TestFieldCacheReopen, RequestHandlersTest, TestAnalyzedSuggestions, LeaderTragicEventTest, TestReqParamsAPI, TestSolrXml, TestDistributedSearch, CollectionTooManyReplicasTest, BooleanFieldTest, BlockJoinFacetRandomTest, DocValuesMissingTest, TimeRoutedAliasUpdateProcessorTest, DataDrivenBlockJoinTest, CdcrReplicationHandlerTest, MBeansHandlerTest, TestNumericTerms32, ExecutePlanActionTest, IndexSizeTriggerTest, TemplateUpdateProcessorTest, TestAtomicUpdateErrorCases, TestIndexingPerformance, TestSolrQueryParser, ShardSplitTest, DistanceUnitsTest, NoCacheHeaderTest, TestUnInvertedFieldException, RecoveryAfterSoftCommitTest, SpatialFilterTest, TestCollectionsAPIViaSolrCloudCluster, TestReplicaProperties, PropertiesRequestHandlerTest, TestHalfAndHalfDocValues, AtomicUpdatesTest, TestTlogReplica, TestUtilizeNode, InfixSuggestersTest, MoreLikeThisHandlerTest, TestFieldTypeResource, SolrCoreCheckLockOnStartupTest, SolrSlf4jReporterTest, TestSimLargeCluster, MoveReplicaHDFSFailoverTest, TestShortCircuitedRequests, TestLuceneMatchVersion, FullHLLTest, HdfsBasicDistributedZkTest, TestMultiValuedNumericRangeQuery, SubstringBytesRefFilterTest, SolrInfoBeanTest, DeleteLastCustomShardedReplicaTest, TestConfigSets, TestIntervalFaceting, DistributedMLTComponentTest, TermVectorComponentDistributedTest, TaggerTest, XmlInterpolationTest, HighlighterConfigTest, HighlighterTest, TestNumericRangeQuery64, SolrShardReporterTest, TestCopyFieldCollectionResource, TestDynamicFieldCollectionResource, TestDynamicFieldResource, TestFieldCollectionResource, TestFieldResource, TestSchemaResource, TestSchemaSimilarityResource, TestSchemaVersionResource, TestManagedStopFilterFactory, TestManagedSynonymGraphFilterFactory, EnumFieldTest, ExternalFileFieldSortTest, IndexSchemaRuntimeFieldTest, IndexSchemaTest, ManagedSchemaRoundRobinCloudTest, PrimitiveFieldTypeTest, RequiredFieldsTest, SchemaApiFailureTest, TestCloudSchemaless, AnalyticsQueryTest, CursorMarkTest, LargeFieldTest, MergeStrategyTest, TestFiltering, TestOverriddenPrefixQueryForCustomFieldType, TestPayloadScoreQParserPlugin, TestQueryTypes, TestReRankQParserPlugin, TestRealTimeGet, TestRecovery, TestRecoveryHdfs, TestSearchPerf, TestSearcherReuse, TestSolr4Spatial2, TestSolrCoreParser, TestSolrFieldCacheBean, TestStandardQParsers, TestStressLucene, TestStressRecovery, TestSurroundQueryParser, TestTrieFacet, TestValueSourceCache, TestXmlQParser, TestXmlQParserPlugin, DistributedFacetSimpleRefinementLongTailTest, RangeFacetCloudTest, TestCloudJSONFacetJoinDomain, TestJsonRequest, SimpleMLTQParserTest, TestBM25SimilarityFactory, TestClassicSimilarityFactory, TestDFISimilarityFactory, TestDFRSimilarityFactory, TestLegacyBM25SimilarityFactory, TestNonDefinedSimilarityFactory, TestPerFieldSimilarity, TestPerFieldSimilarityWithDefaultOverride, TestSweetSpotSimilarityFactory, BasicAuthStandaloneTest, PKIAuthenticationIntegrationTest, TestAuthorizationFramework, TestPKIAuthenticationPlugin, TestRuleBasedAuthorizationPlugin, TestSha256AuthenticationProvider, TestZkAclsWithHadoopAuth, CacheHeaderTest, SolrRequestParserTest, FileBasedSpellCheckerTest, SpellCheckCollatorTest, SpellingQueryConverterTest, SuggesterTest, TestFileDictionaryLookup, TestFuzzyAnalyzedSuggestions, TestPhraseSuggestions, BlockDirectoryTest, HdfsLockFactoryTest, TestDocTermOrds, TestDocTermOrdsUninvertLimit, RootFieldTest]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=RootFieldTest -Dtests.seed=CE7B6F721C0D523A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.badapples=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ko -Dtests.timezone=Africa/Dakar -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J1 | RootFieldTest (suite) <<<
[junit4] > Throwable #1: java.lang.OutOfMemoryError: Java heap space
[junit4] Completed [822/837 (4!)] on J1 in 59.33s, 0 tests, 1 error <<< FAILURES!
[...truncated 43 lines...]
[junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20190111_191217_63385766783445832503.sysout
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/heapdumps/java_pid9012.hprof ...
[junit4] Heap dump file created [457414437 bytes in 1.048 secs]
[junit4] <<< JVM J1: EOF ----
[...truncated 8915 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/build.xml:661: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid9012.hprof
Total time: 331 minutes 46 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-BadApples-NightlyTests-master - Build # 44 -
Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-master/44/
22 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.RestartWhileUpdatingTest
Error Message:
ObjectTracker found 30 object(s) that were not released!!! [MMapDirectory, MDCAwareThreadPoolExecutor, SolrCore, MMapDirectory, MMapDirectory, SolrIndexSearcher, MDCAwareThreadPoolExecutor, SolrIndexSearcher, TransactionLog, MMapDirectory, MDCAwareThreadPoolExecutor, MMapDirectory, MMapDirectory, MMapDirectory, TransactionLog, InternalHttpClient, MMapDirectory, MMapDirectory, SolrIndexSearcher, SolrCore, TransactionLog, MDCAwareThreadPoolExecutor, SolrIndexSearcher, MMapDirectory, MMapDirectory, SolrCore, MMapDirectory, SolrCore, TransactionLog, TransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:901) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1054) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:307) at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2105) at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257) at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1106) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:993) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:901) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:307) at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2105) at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257) at org.apache.solr.handler.IndexFetcher.openNewSearcherAndUpdateCommitPoint(IndexFetcher.java:896) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:653) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:346) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424) at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237) at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:652) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:326) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188) at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468) at org.apache.solr.update.UpdateLog.init(UpdateLog.java:388) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:161) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:116) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:120) at sun.reflect.GeneratedConstructorAccessor127.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:804) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:866) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1123) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:991) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:901) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95) at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:257) at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:131) at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2098) at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257) at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1106) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:993) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:770) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188) at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468) at org.apache.solr.update.UpdateLog.init(UpdateLog.java:388) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:161) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:116) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:120) at sun.reflect.GeneratedConstructorAccessor127.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:804) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:866) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1123) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:991) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330) at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225) at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420) at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237) at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:652) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:326) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95) at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:257) at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:131) at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2098) at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257) at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1106) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:993) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:307) at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2084) at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257) at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1994) at org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:708) at org.apache.solr.update.processor.RunUpdateProcessor.processCommit(RunUpdateProcessorFactory.java:93) at org.apache.solr.update.processor.UpdateRequestProcessor.processCommit(UpdateRequestProcessor.java:68) at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalCommit(DistributedUpdateProcessor.java:2033) at org.apache.solr.update.processor.DistributedUpdateProcessor.processCommit(DistributedUpdateProcessor.java:2017) at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processCommit(LogUpdateProcessorFactory.java:160) at org.apache.solr.handler.RequestHandlerUtils.handleCommit(RequestHandlerUtils.java:69) at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:62) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551) at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:711) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:516) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1054) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188) at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468) at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1329) at org.apache.solr.update.UpdateLog.add(UpdateLog.java:572) at org.apache.solr.update.UpdateLog.add(UpdateLog.java:552) at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:351) at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:289) at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:236) at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67) at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55) at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:994) at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1215) at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:699) at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103) at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144) at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:332) at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130) at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:297) at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277) at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:190) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195) at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:121) at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70) at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97) at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551) at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:711) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:516) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:901) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:307) at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2105) at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257) at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1106) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:993) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95) at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:257) at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:131) at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2098) at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257) at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1106) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:993) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1054) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1054) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188) at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468) at org.apache.solr.update.UpdateLog.init(UpdateLog.java:388) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:161) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:116) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:120) at sun.reflect.GeneratedConstructorAccessor127.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:804) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:866) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1123) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:991) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188) at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468) at org.apache.solr.update.UpdateLog.init(UpdateLog.java:388) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:161) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:116) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:120) at sun.reflect.GeneratedConstructorAccessor127.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:804) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:866) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1123) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:991) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) expected null, but was:<ObjectTracker found 30 object(s) that were not released!!! [MMapDirectory, MDCAwareThreadPoolExecutor, SolrCore, MMapDirectory, MMapDirectory, SolrIndexSearcher, MDCAwareThreadPoolExecutor, SolrIndexSearcher, TransactionLog, MMapDirectory, MDCAwareThreadPoolExecutor, MMapDirectory, MMapDirectory, MMapDirectory, TransactionLog, InternalHttpClient, MMapDirectory, MMapDirectory, SolrIndexSearcher, SolrCore, TransactionLog, MDCAwareThreadPoolExecutor, SolrIndexSearcher, MMapDirectory, MMapDirectory, SolrCore, MMapDirectory, SolrCore, TransactionLog, TransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:901) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1054) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:307) at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2105) at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257) at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1106) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:993) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:901) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:307) at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2105) at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257) at org.apache.solr.handler.IndexFetcher.openNewSearcherAndUpdateCommitPoint(IndexFetcher.java:896) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:653) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:346) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424) at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237) at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:652) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:326) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188) at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468) at org.apache.solr.update.UpdateLog.init(UpdateLog.java:388) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:161) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:116) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:120) at sun.reflect.GeneratedConstructorAccessor127.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:804) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:866) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1123) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:991) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:901) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95) at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:257) at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:131) at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2098) at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257) at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1106) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:993) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:770) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188) at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468) at org.apache.solr.update.UpdateLog.init(UpdateLog.java:388) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:161) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:116) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:120) at sun.reflect.GeneratedConstructorAccessor127.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:804) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:866) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1123) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:991) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330) at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225) at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420) at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237) at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:652) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:326) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95) at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:257) at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:131) at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2098) at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257) at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1106) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:993) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.So
[...truncated too long message...]
SchemaConcurrent, AnalysisErrorHandlingTest, TestCollationField, SolrXmlInZkTest, TestHdfsCloudBackupRestore, TestLFUCache, RollingRestartTest, SuggestComponentTest, TestEmbeddedSolrServerSchemaAPI, ResourceLoaderTest, V2ApiIntegrationTest, DistributedFacetSimpleRefinementLongTailTest, BitVectorTest, ClusterStateTest, TestSystemIdResolver, TestSchemaVersionResource, UpdateLogTest, ExternalFileFieldSortTest, CustomCollectionTest, TestDocBasedVersionConstraints, TestRestoreCore, DistributedFacetPivotSmallAdvancedTest, TestCSVResponseWriter, TestSolrFieldCacheBean, TestCollectionAPIs, RecoveryZkTest, SolrTestCaseJ4Test, TestNestedDocsSort, LoggingHandlerTest, OpenExchangeRatesOrgProviderTest, TestInitParams, CoreAdminCreateDiscoverTest, SolrIndexMetricsTest, TestCharFilters, TriLevelCompositeIdRoutingTest]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TriLevelCompositeIdRoutingTest -Dtests.seed=C1CE3DEC1B9E220D -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.badapples=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=es-US -Dtests.timezone=SystemV/MST7MDT -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J2 | TriLevelCompositeIdRoutingTest (suite) <<<
[junit4] > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
[junit4] > at __randomizedtesting.SeedInfo.seed([C1CE3DEC1B9E220D]:0)Throwable #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=19215, name=h2sc-6017-thread-5, state=RUNNABLE, group=TGRP-TriLevelCompositeIdRoutingTest]
[junit4] > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceededThrowable #3: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=19084, name=h2sc-5852-thread-8, state=RUNNABLE, group=TGRP-TriLevelCompositeIdRoutingTest]
[junit4] > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceededThrowable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=18458, name=Connection evictor, state=RUNNABLE, group=TGRP-TriLevelCompositeIdRoutingTest]
[junit4] > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceededThrowable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=19480, name=h2sc-5791-thread-31, state=RUNNABLE, group=TGRP-TriLevelCompositeIdRoutingTest]
[junit4] > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceededThrowable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=19274, name=h2sc-6017-thread-13, state=RUNNABLE, group=TGRP-TriLevelCompositeIdRoutingTest]
[junit4] > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceededThrowable #7: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=19231, name=h2sc-5845-thread-6, state=RUNNABLE, group=TGRP-TriLevelCompositeIdRoutingTest]
[junit4] > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
[junit4] > at __randomizedtesting.SeedInfo.seed([C1CE3DEC1B9E220D]:0)
[junit4] > at java.util.Collections.unmodifiableMap(Collections.java:1433)
[junit4] > at org.apache.logging.log4j.spi.DefaultThreadContextMap.put(DefaultThreadContextMap.java:90)
[junit4] > at org.apache.logging.log4j.ThreadContext.put(ThreadContext.java:246)
[junit4] > at org.apache.logging.slf4j.Log4jMDCAdapter.setContextMap(Log4jMDCAdapter.java:59)
[junit4] > at org.slf4j.MDC.setContextMap(MDC.java:264)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:203)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$38/590464868.run(Unknown Source)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)Throwable #8: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=18394, name=Connection evictor, state=RUNNABLE, group=TGRP-TriLevelCompositeIdRoutingTest]
[junit4] > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceededThrowable #9: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=17814, name=Connection evictor, state=RUNNABLE, group=TGRP-TriLevelCompositeIdRoutingTest]
[junit4] > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceededThrowable #10: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=18066, name=Connection evictor, state=RUNNABLE, group=TGRP-TriLevelCompositeIdRoutingTest]
[junit4] > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceededThrowable #11: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=19282, name=h2sc-6017-thread-17, state=RUNNABLE, group=TGRP-TriLevelCompositeIdRoutingTest]
[junit4] > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceededThrowable #12: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=18331, name=Connection evictor, state=RUNNABLE, group=TGRP-TriLevelCompositeIdRoutingTest]
[junit4] > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceededThrowable #13: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=18353, name=Connection evictor, state=RUNNABLE, group=TGRP-TriLevelCompositeIdRoutingTest]
[junit4] > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceededThrowable #14: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=18288, name=Scheduler-1399113449, state=RUNNABLE, group=TGRP-TriLevelCompositeIdRoutingTest]
[junit4] > Caused by: java.lang.IllegalMonitorStateException
[junit4] > at __randomizedtesting.SeedInfo.seed([C1CE3DEC1B9E220D]:0)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.signal(AbstractQueuedSynchronizer.java:1939)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1103)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)Throwable #15: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=18795, name=h2sc-6034-thread-7, state=RUNNABLE, group=TGRP-TriLevelCompositeIdRoutingTest]
[junit4] > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceededThrowable #16: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=20140, name=zkCallback-6105-thread-5, state=RUNNABLE, group=TGRP-TriLevelCompositeIdRoutingTest]
[junit4] > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
[junit4] Completed [741/837 (4!)] on J2 in 7211.71s, 1 test, 17 errors <<< FAILURES!
[...truncated 316 lines...]
[junit4] JVM J2: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20190104_192545_0246134492540108072256.sysout
[junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
[junit4] Dumping heap to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/heapdumps/java_pid4736.hprof ...
[junit4] Heap dump file created [860149394 bytes in 40.172 secs]
[junit4] <<< JVM J2: EOF ----
[junit4] JVM J2: stderr was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20190104_192545_0247869809209580270080.syserr
[junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
[junit4]
[junit4] Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "closeThreadPool-5675-thread-3-SendThread(lucene2-us-west.apache.org:39375)"
[junit4]
[junit4] Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "zkCallback-6105-thread-5"
[junit4] <<< JVM J2: EOF ----
[...truncated 8930 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/build.xml:661: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid4736.hprof
Total time: 597 minutes 56 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-BadApples-NightlyTests-master - Build # 43 -
Still unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-master/43/
7 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.testDeleteNode
Error Message:
unexpected DELETENODE status: {responseHeader={status=0,QTime=72},status={state=notfound,msg=Did not find [search_rate_trigger3/64cbf20efcd8daT6ka0yhexlcekhqbz8vdd4r05c/0] in any tasks queue}}
Stack Trace:
java.lang.AssertionError: unexpected DELETENODE status: {responseHeader={status=0,QTime=72},status={state=notfound,msg=Did not find [search_rate_trigger3/64cbf20efcd8daT6ka0yhexlcekhqbz8vdd4r05c/0] in any tasks queue}}
at __randomizedtesting.SeedInfo.seed([988D9624F13AA4D8:BA1F58A6C6F02BA5]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.lambda$testDeleteNode$5(SearchRateTriggerIntegrationTest.java:693)
at java.util.ArrayList.forEach(ArrayList.java:1257)
at org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest.testDeleteNode(SearchRateTriggerIntegrationTest.java:677)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testNodeMarkersRegistration
Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:10036_solr should have been deleted
Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:10036_solr should have been deleted
at __randomizedtesting.SeedInfo.seed([988D9624F13AA4D8:80371E28FF0F6937]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at org.junit.Assert.assertFalse(Assert.java:64)
at org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testNodeMarkersRegistration(TestSimTriggerIntegration.java:892)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [NRTCachingDirectory] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:770) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) expected null, but was:<ObjectTracker found 1 object(s) that were not released!!! [NRTCachingDirectory] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:770) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) >
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [NRTCachingDirectory]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)
at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:770)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)
at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at org.eclipse.jetty.server.Server.handle(Server.java:502)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
expected null, but was:<ObjectTracker found 1 object(s) that were not released!!! [NRTCachingDirectory]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)
at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:770)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)
at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at org.eclipse.jetty.server.Server.handle(Server.java:502)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
>
at __randomizedtesting.SeedInfo.seed([988D9624F13AA4D8]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.failNotNull(Assert.java:755)
at org.junit.Assert.assertNull(Assert.java:737)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:319)
at sun.reflect.GeneratedMethodAccessor68.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:901)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
Error Message:
ObjectTracker found 58 object(s) that were not released!!! [MDCAwareThreadPoolExecutor, MMapDirectory, MMapDirectory, SolrIndexSearcher, SolrIndexSearcher, MMapDirectory, MMapDirectory, SolrCore, TransactionLog, MMapDirectory, MMapDirectory, TransactionLog, MMapDirectory, TransactionLog, TransactionLog, InternalHttpClient, SolrCore, TransactionLog, MMapDirectory, SolrIndexSearcher, InternalHttpClient, MMapDirectory, SolrIndexSearcher, MMapDirectory, MDCAwareThreadPoolExecutor, MMapDirectory, SolrCore, MMapDirectory, MMapDirectory, TransactionLog, MMapDirectory, TransactionLog, MMapDirectory, TransactionLog, MMapDirectory, MMapDirectory, SolrCore, SolrCore, MDCAwareThreadPoolExecutor, MMapDirectory, MDCAwareThreadPoolExecutor, TransactionLog, SolrCore, TransactionLog, SolrIndexSearcher, MDCAwareThreadPoolExecutor, MMapDirectory, SolrIndexSearcher, InternalHttpClient, MDCAwareThreadPoolExecutor, SolrIndexSearcher, MMapDirectory, SolrIndexSearcher, InternalHttpClient, MMapDirectory, SolrCore, MDCAwareThreadPoolExecutor, TransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:901) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:770) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:307) at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2084) at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257) at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1994) at org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:708) at org.apache.solr.update.UpdateLog$LogReplayer.doReplay(UpdateLog.java:1954) at org.apache.solr.update.UpdateLog$LogReplayer.run(UpdateLog.java:1749) at org.apache.solr.update.UpdateLog.lambda$applyBufferedUpdates$5(UpdateLog.java:1690) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:307) at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2105) at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257) at org.apache.solr.handler.IndexFetcher.openNewSearcherAndUpdateCommitPoint(IndexFetcher.java:896) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:653) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:346) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424) at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237) at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:652) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:326) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:359) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:738) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1054) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188) at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468) at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1329) at org.apache.solr.update.UpdateLog.add(UpdateLog.java:572) at org.apache.solr.update.UpdateLog.add(UpdateLog.java:552) at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:351) at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:289) at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:236) at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67) at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55) at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:994) at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1215) at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:699) at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103) at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144) at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:332) at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130) at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:297) at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277) at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:190) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195) at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:121) at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70) at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97) at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551) at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:710) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95) at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:257) at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:131) at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2098) at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257) at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1106) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:993) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:770) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188) at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468) at org.apache.solr.update.UpdateLog.init(UpdateLog.java:388) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:161) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:116) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:120) at sun.reflect.GeneratedConstructorAccessor155.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:804) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:866) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1123) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:991) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:508) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188) at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468) at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1329) at org.apache.solr.update.UpdateLog.add(UpdateLog.java:572) at org.apache.solr.update.UpdateLog.add(UpdateLog.java:552) at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:351) at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:289) at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:236) at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67) at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55) at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:994) at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1215) at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:699) at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103) at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144) at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:332) at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130) at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:297) at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277) at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:190) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195) at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:121) at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70) at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97) at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551) at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:710) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188) at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468) at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1329) at org.apache.solr.update.UpdateLog.add(UpdateLog.java:572) at org.apache.solr.update.UpdateLog.add(UpdateLog.java:552) at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:351) at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:289) at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:236) at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67) at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55) at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:994) at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1215) at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:699) at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:103) at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:110) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144) at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:332) at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130) at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:297) at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:277) at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:190) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195) at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:121) at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:70) at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97) at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.core.SolrCore.execute(SolrCore.java:2551) at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:710) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330) at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225) at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420) at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237) at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:652) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:326) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1054) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:188) at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:468) at org.apache.solr.update.UpdateLog.init(UpdateLog.java:388) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:161) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:116) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:120) at sun.reflect.GeneratedConstructorAccessor155.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:804) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:866) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1123) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:991) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:697) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MMapDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:770) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:967) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:874) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:502) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.search.SolrIndexSearcher at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:307) at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2084) at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2257) at org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:700) at org.apache.solr.update.CommitTracker.run(CommitTracker.java:270) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330) at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225) at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:267) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420) at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237) at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:652) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:326) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.a
[...truncated too long message...]
g settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null1645212367
[copy] Copying 256 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null1645212367
[delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/null1645212367
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 48ms :: artifacts dl 3ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 575 minutes 26 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$34.hasMatch(FilePath.java:2678)
at hudson.FilePath$34.invoke(FilePath.java:2557)
at hudson.FilePath$34.invoke(FilePath.java:2547)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1036)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1038)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
[JENKINS] Lucene-Solr-BadApples-NightlyTests-master - Build # 42 -
Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-master/42/
6 tests failed.
FAILED: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
at java.util.Arrays.copyOf(Arrays.java:3332)
at java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:124)
at java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:649)
at java.lang.StringBuilder.append(StringBuilder.java:202)
at org.apache.http.client.utils.URLEncodedUtils.urlEncode(URLEncodedUtils.java:539)
at org.apache.http.client.utils.URLEncodedUtils.encodeFormFields(URLEncodedUtils.java:655)
at org.apache.http.client.utils.URLEncodedUtils.format(URLEncodedUtils.java:403)
at org.apache.http.client.utils.URLEncodedUtils.format(URLEncodedUtils.java:381)
at org.apache.http.client.entity.UrlEncodedFormEntity.<init>(UrlEncodedFormEntity.java:75)
at org.apache.solr.client.solrj.impl.HttpSolrClient.fillContentStream(HttpSolrClient.java:514)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:421)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:254)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:245)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:213)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1110)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:1015)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:1031)
at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:228)
at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:167)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:669)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:153)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
FAILED: org.apache.solr.cloud.RestartWhileUpdatingTest.test
Error Message:
There are still nodes recoverying - waited for 320 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
at __randomizedtesting.SeedInfo.seed([5EB41C54616B95B7:D6E0238ECF97F84F]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:195)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1038)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1595)
at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:143)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1070)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1042)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
Stack Trace:
java.lang.AssertionError
at __randomizedtesting.SeedInfo.seed([5EB41C54616B95B7:D6E0238ECF97F84F]:0)
at org.junit.Assert.fail(Assert.java:86)
at org.junit.Assert.assertTrue(Assert.java:41)
at org.junit.Assert.assertTrue(Assert.java:52)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:228)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:148)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1070)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1042)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TriLevelCompositeIdRoutingTest.test
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
FAILED: org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest.test
Error Message:
There are still nodes recoverying - waited for 320 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
at __randomizedtesting.SeedInfo.seed([5EB41C54616B95B7:D6E0238ECF97F84F]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:195)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1038)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1595)
at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:143)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1070)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1042)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.update.TestInPlaceUpdatesDistrib.test
Error Message:
Error from server at https://127.0.0.1:34238/w_fob/b/collection1: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/index-SimpleFSDirectory-003/_n_3_Lucene70_1.dvm: Too many open files
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:34238/w_fob/b/collection1: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/index-SimpleFSDirectory-003/_n_3_Lucene70_1.dvm: Too many open files
at __randomizedtesting.SeedInfo.seed([5EB41C54616B95B7:D6E0238ECF97F84F]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:650)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:256)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:245)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:213)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1110)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:504)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:479)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1726)
at org.apache.solr.update.TestInPlaceUpdatesDistrib.docValuesUpdateTest(TestInPlaceUpdatesDistrib.java:408)
at org.apache.solr.update.TestInPlaceUpdatesDistrib.test(TestInPlaceUpdatesDistrib.java:161)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1070)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1042)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13566 lines...]
[junit4] Suite: org.apache.solr.update.TestInPlaceUpdatesDistrib
[junit4] 2> 338944 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/init-core-data-001
[junit4] 2> 338945 WARN (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 338946 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 338950 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 338951 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /w_fob/b
[junit4] 2> 338954 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.SolrTestCaseJ4 updateLog impl=solr.UpdateLog
[junit4] 2> 338954 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore
[junit4] 2> 338955 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 338992 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 339006 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.s.IndexSchema [null] Schema name=inplace-updates
[junit4] 2> 339011 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.s.IndexSchema Loaded schema inplace-updates/1.6 with uniqueid field id
[junit4] 2> 339142 WARN (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 339147 WARN (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@487e9d9[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 339151 WARN (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 339152 WARN (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@5c9fb3dc[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 339185 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 339185 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 339209 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a
[junit4] 2> 339224 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a
[junit4] 2> 339225 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a
[junit4] 2> 339229 INFO (coreLoadExecutor-1838-thread-1) [ x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 339281 INFO (coreLoadExecutor-1838-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 339300 INFO (coreLoadExecutor-1838-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=inplace-updates
[junit4] 2> 339349 INFO (coreLoadExecutor-1838-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema inplace-updates/1.6 with uniqueid field id
[junit4] 2> 339349 INFO (coreLoadExecutor-1838-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1, trusted=true
[junit4] 2> 339349 INFO (coreLoadExecutor-1838-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a
[junit4] 2> 339349 INFO (coreLoadExecutor-1838-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/init-core-data-001/]
[junit4] 2> 339398 WARN (coreLoadExecutor-1838-thread-1) [ x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 339530 INFO (coreLoadExecutor-1838-thread-1) [ x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 339530 INFO (coreLoadExecutor-1838-thread-1) [ x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 339533 INFO (coreLoadExecutor-1838-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 339533 INFO (coreLoadExecutor-1838-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 339536 INFO (coreLoadExecutor-1838-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@526c8aa4[collection1] main]
[junit4] 2> 339536 WARN (coreLoadExecutor-1838-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
[junit4] 2> 339537 INFO (coreLoadExecutor-1838-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 339537 INFO (coreLoadExecutor-1838-thread-1) [ x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1620499668974698496
[junit4] 2> 339540 INFO (searcherExecutor-1839-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@526c8aa4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 339542 INFO (SUITE-TestInPlaceUpdatesDistrib-seed#[5EB41C54616B95B7]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore end
[junit4] 2> 339557 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 339560 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 339560 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 339660 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ZkTestServer start zk server on port:42667
[junit4] 2> 339660 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:42667
[junit4] 2> 339660 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 42667
[junit4] 2> 339682 INFO (zkConnectionManagerCallback-2041-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 339705 INFO (zkConnectionManagerCallback-2043-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 339707 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 339722 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema-inplace-updates.xml to /configs/conf1/schema.xml
[junit4] 2> 339724 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 339725 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 339727 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 339728 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 339729 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 339730 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 339731 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 339732 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 339733 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ZkTestServer put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 339814 INFO (zkConnectionManagerCallback-2047-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 339817 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 340209 WARN (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 340217 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 340217 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 340217 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 340255 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 340255 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 340255 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 340262 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@44c43f9{/w_fob/b,null,AVAILABLE}
[junit4] 2> 340262 WARN (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@453a2267[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 340263 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@273544f0{SSL,[ssl, http/1.1]}{127.0.0.1:38865}
[junit4] 2> 340263 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.e.j.s.Server Started @340326ms
[junit4] 2> 340263 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/tempDir-001/control/data, hostContext=/w_fob/b, hostPort=38865, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/control-001/cores}
[junit4] 2> 340264 ERROR (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 340264 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 340264 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 340264 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 340264 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 340264 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-12-21T21:48:34.773Z
[junit4] 2> 340308 INFO (zkConnectionManagerCallback-2049-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 340313 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 340313 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/control-001/solr.xml
[junit4] 2> 340318 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 340318 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 340320 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 340424 WARN (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 340425 WARN (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@3c60aacf[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 340429 WARN (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 340430 WARN (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@709a10f4[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 340431 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42667/solr
[junit4] 2> 340434 INFO (zkConnectionManagerCallback-2055-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 340437 INFO (zkConnectionManagerCallback-2057-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 340614 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38865_w_fob%2Fb
[junit4] 2> 340615 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.Overseer Overseer (id=72487752055652357-127.0.0.1:38865_w_fob%2Fb-n_0000000000) starting
[junit4] 2> 340623 INFO (zkConnectionManagerCallback-2064-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 340678 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42667/solr ready
[junit4] 2> 340690 INFO (OverseerStateUpdate-72487752055652357-127.0.0.1:38865_w_fob%2Fb-n_0000000000) [n:127.0.0.1:38865_w_fob%2Fb ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:38865_w_fob%2Fb
[junit4] 2> 340698 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38865_w_fob%2Fb
[junit4] 2> 340720 INFO (zkCallback-2056-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 340723 INFO (zkCallback-2063-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 340766 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 340799 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a
[junit4] 2> 340855 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a
[junit4] 2> 340855 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a
[junit4] 2> 340858 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/control-001/cores
[junit4] 2> 340936 INFO (zkConnectionManagerCallback-2070-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 340938 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 340939 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42667/solr ready
[junit4] 2> 340984 INFO (qtp1908906739-6525) [n:127.0.0.1:38865_w_fob%2Fb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38865_w_fob%252Fb&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 341016 INFO (OverseerThreadFactory-1853-thread-1) [ ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 341127 INFO (qtp1908906739-6523) [n:127.0.0.1:38865_w_fob%2Fb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
[junit4] 2> 341130 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 341138 INFO (qtp1908906739-6526) [n:127.0.0.1:38865_w_fob%2Fb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 341139 INFO (qtp1908906739-6522) [n:127.0.0.1:38865_w_fob%2Fb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 341140 INFO (qtp1908906739-6523) [n:127.0.0.1:38865_w_fob%2Fb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 341180 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 341181 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 342238 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 342320 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=inplace-updates
[junit4] 2> 342325 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema inplace-updates/1.6 with uniqueid field id
[junit4] 2> 342325 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 342326 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a
[junit4] 2> 342326 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 342354 WARN (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 342491 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 342491 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 342504 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 342505 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 342509 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6ecc00b7[control_collection_shard1_replica_n1] main]
[junit4] 2> 342513 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 342514 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 342515 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 342515 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1620499672097357824
[junit4] 2> 342535 INFO (searcherExecutor-1858-thread-1-processing-n:127.0.0.1:38865_w_fob%2Fb x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@6ecc00b7[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 342552 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 342552 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
[junit4] 2> 342561 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 342561 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 342561 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:38865/w_fob/b/control_collection_shard1_replica_n1/
[junit4] 2> 342562 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 342562 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:38865/w_fob/b/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 342562 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72487752055652357-core_node2-n_0000000000
[junit4] 2> 342565 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:38865/w_fob/b/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 342670 INFO (zkCallback-2056-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 342672 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 342685 INFO (qtp1908906739-6524) [n:127.0.0.1:38865_w_fob%2Fb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1504
[junit4] 2> 342714 INFO (zkCallback-2056-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 342718 INFO (qtp1908906739-6525) [n:127.0.0.1:38865_w_fob%2Fb ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 342775 INFO (zkCallback-2056-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 342775 INFO (zkCallback-2056-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 342776 INFO (qtp1908906739-6525) [n:127.0.0.1:38865_w_fob%2Fb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38865_w_fob%252Fb&wt=javabin&version=2} status=0 QTime=1792
[junit4] 2> 342815 INFO (zkCallback-2056-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 342899 INFO (zkConnectionManagerCallback-2075-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 342906 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 342907 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42667/solr ready
[junit4] 2> 342930 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 342945 INFO (qtp1908906739-6525) [n:127.0.0.1:38865_w_fob%2Fb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 342981 INFO (OverseerCollectionConfigSetProcessor-72487752055652357-127.0.0.1:38865_w_fob%2Fb-n_0000000000) [ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 342992 INFO (OverseerThreadFactory-1853-thread-2) [ ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 343473 WARN (OverseerThreadFactory-1853-thread-2) [ ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 343478 INFO (qtp1908906739-6525) [n:127.0.0.1:38865_w_fob%2Fb ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 343479 INFO (qtp1908906739-6525) [n:127.0.0.1:38865_w_fob%2Fb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=534
[junit4] 2> 343500 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=3
[junit4] 2> 343899 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/shard-1-001 of type NRT
[junit4] 2> 343934 WARN (closeThreadPool-2076-thread-1) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 343934 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 343934 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 343934 INFO (closeThreadPool-2076-thread-1) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 343936 INFO (closeThreadPool-2076-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 343936 INFO (closeThreadPool-2076-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 343936 INFO (closeThreadPool-2076-thread-1) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 343936 INFO (closeThreadPool-2076-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@25d687d6{/w_fob/b,null,AVAILABLE}
[junit4] 2> 343937 WARN (closeThreadPool-2076-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@246f33f1[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 343938 INFO (closeThreadPool-2076-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@38e392b9{SSL,[ssl, http/1.1]}{127.0.0.1:34300}
[junit4] 2> 343938 INFO (closeThreadPool-2076-thread-1) [ ] o.e.j.s.Server Started @344000ms
[junit4] 2> 343952 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/w_fob/b, hostPort=34300, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/shard-1-001/cores}
[junit4] 2> 343952 ERROR (closeThreadPool-2076-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 343952 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 343953 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 343953 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 343953 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 343953 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-12-21T21:48:38.462Z
[junit4] 2> 344006 INFO (zkConnectionManagerCallback-2078-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 344008 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 344008 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/shard-1-001/solr.xml
[junit4] 2> 344014 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 344014 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 344016 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 344110 WARN (closeThreadPool-2076-thread-1) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 344157 WARN (closeThreadPool-2076-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@55262713[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 344259 WARN (closeThreadPool-2076-thread-1) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 344265 WARN (closeThreadPool-2076-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@36f11526[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 344266 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42667/solr
[junit4] 2> 344300 INFO (zkConnectionManagerCallback-2084-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 344319 INFO (zkConnectionManagerCallback-2086-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 344332 INFO (closeThreadPool-2076-thread-1) [n:127.0.0.1:34300_w_fob%2Fb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 344337 INFO (closeThreadPool-2076-thread-1) [n:127.0.0.1:34300_w_fob%2Fb ] o.a.s.c.ZkController Publish node=127.0.0.1:34300_w_fob%2Fb as DOWN
[junit4] 2> 344338 INFO (closeThreadPool-2076-thread-1) [n:127.0.0.1:34300_w_fob%2Fb ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 344338 INFO (closeThreadPool-2076-thread-1) [n:127.0.0.1:34300_w_fob%2Fb ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34300_w_fob%2Fb
[junit4] 2> 344340 INFO (zkCallback-2056-thread-3) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 344340 INFO (zkCallback-2063-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 344345 INFO (zkCallback-2074-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 344381 INFO (zkCallback-2085-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 344436 INFO (zkConnectionManagerCallback-2093-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 344438 INFO (closeThreadPool-2076-thread-1) [n:127.0.0.1:34300_w_fob%2Fb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 344440 INFO (closeThreadPool-2076-thread-1) [n:127.0.0.1:34300_w_fob%2Fb ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42667/solr ready
[junit4] 2> 344459 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/shard-2-001 of type NRT
[junit4] 2> 344513 WARN (closeThreadPool-2076-thread-2) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 344514 INFO (closeThreadPool-2076-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 344514 INFO (closeThreadPool-2076-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 344514 INFO (closeThreadPool-2076-thread-2) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 344518 INFO (closeThreadPool-2076-thread-1) [n:127.0.0.1:34300_w_fob%2Fb ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 344619 INFO (closeThreadPool-2076-thread-1) [n:127.0.0.1:34300_w_fob%2Fb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a
[junit4] 2> 344636 INFO (closeThreadPool-2076-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 344636 INFO (closeThreadPool-2076-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 344636 INFO (closeThreadPool-2076-thread-2) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 344640 INFO (closeThreadPool-2076-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6761aeb2{/w_fob/b,null,AVAILABLE}
[junit4] 2> 344640 WARN (closeThreadPool-2076-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@5b2ae09f[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 344641 INFO (closeThreadPool-2076-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@65aa4360{SSL,[ssl, http/1.1]}{127.0.0.1:34238}
[junit4] 2> 344641 INFO (closeThreadPool-2076-thread-2) [ ] o.e.j.s.Server Started @344703ms
[junit4] 2> 344641 INFO (closeThreadPool-2076-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/tempDir-001/jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/w_fob/b, hostPort=34238, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/shard-2-001/cores}
[junit4] 2> 344641 ERROR (closeThreadPool-2076-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 344641 INFO (closeThreadPool-2076-thread-2) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 344641 INFO (closeThreadPool-2076-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 344642 INFO (closeThreadPool-2076-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 344642 INFO (closeThreadPool-2076-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 344642 INFO (closeThreadPool-2076-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-12-21T21:48:39.151Z
[junit4] 2> 344658 INFO (closeThreadPool-2076-thread-1) [n:127.0.0.1:34300_w_fob%2Fb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a
[junit4] 2> 344658 INFO (closeThreadPool-2076-thread-1) [n:127.0.0.1:34300_w_fob%2Fb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a
[junit4] 2> 344664 INFO (zkConnectionManagerCallback-2096-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 344665 INFO (closeThreadPool-2076-thread-2) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 344665 INFO (closeThreadPool-2076-thread-2) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/shard-2-001/solr.xml
[junit4] 2> 344671 INFO (closeThreadPool-2076-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 344671 INFO (closeThreadPool-2076-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 344673 INFO (closeThreadPool-2076-thread-2) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 344676 INFO (closeThreadPool-2076-thread-1) [n:127.0.0.1:34300_w_fob%2Fb ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/shard-1-001/cores
[junit4] 2> 344987 INFO (OverseerCollectionConfigSetProcessor-72487752055652357-127.0.0.1:38865_w_fob%2Fb-n_0000000000) [ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 345002 INFO (TEST-TestInPlaceUpdatesDistrib.test-seed#[5EB41C54616B95B7]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/shard-3-001 of type NRT
[junit4] 2> 345003 WARN (closeThreadPool-2076-thread-1) [ ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
[junit4] 2> 345007 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
[junit4] 2> 345007 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 345007 INFO (closeThreadPool-2076-thread-1) [ ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
[junit4] 2> 345041 INFO (closeThreadPool-2076-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 345041 INFO (closeThreadPool-2076-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 345041 INFO (closeThreadPool-2076-thread-1) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 345041 INFO (closeThreadPool-2076-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@e950053{/w_fob/b,null,AVAILABLE}
[junit4] 2> 345042 WARN (closeThreadPool-2076-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@5e098f31[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 345042 INFO (closeThreadPool-2076-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@21f16915{SSL,[ssl, http/1.1]}{127.0.0.1:34046}
[junit4] 2> 345042 INFO (closeThreadPool-2076-thread-1) [ ] o.e.j.s.Server Started @345105ms
[junit4] 2> 345043 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/tempDir-001/jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/w_fob/b, hostPort=34046, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/shard-3-001/cores}
[junit4] 2> 345043 ERROR (closeThreadPool-2076-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 345043 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 345043 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 345043 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 345043 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 345043 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-12-21T21:48:39.552Z
[junit4] 2> 345069 INFO (zkConnectionManagerCallback-2098-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 345070 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 345070 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/shard-3-001/solr.xml
[junit4] 2> 345077 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 345077 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 345079 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 345097 WARN (closeThreadPool-2076-thread-2) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 345142 WARN (closeThreadPool-2076-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@56520a7f[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 345153 WARN (closeThreadPool-2076-thread-2) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 345171 WARN (closeThreadPool-2076-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@54bbe5e5[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 345173 INFO (closeThreadPool-2076-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42667/solr
[junit4] 2> 345196 INFO (zkConnectionManagerCallback-2104-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 345243 INFO (zkConnectionManagerCallback-2106-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 345251 INFO (closeThreadPool-2076-thread-2) [n:127.0.0.1:34238_w_fob%2Fb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 345256 INFO (closeThreadPool-2076-thread-2) [n:127.0.0.1:34238_w_fob%2Fb ] o.a.s.c.ZkController Publish node=127.0.0.1:34238_w_fob%2Fb as DOWN
[junit4] 2> 345257 INFO (closeThreadPool-2076-thread-2) [n:127.0.0.1:34238_w_fob%2Fb ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 345257 INFO (closeThreadPool-2076-thread-2) [n:127.0.0.1:34238_w_fob%2Fb ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34238_w_fob%2Fb
[junit4] 2> 345258 INFO (zkCallback-2074-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 345259 INFO (zkCallback-2085-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 345533 INFO (zkCallback-2092-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 345533 INFO (zkCallback-2056-thread-3) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 345533 INFO (zkCallback-2063-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 345534 INFO (zkCallback-2105-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 345572 INFO (zkConnectionManagerCallback-2113-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 345574 INFO (closeThreadPool-2076-thread-2) [n:127.0.0.1:34238_w_fob%2Fb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 345575 INFO (closeThreadPool-2076-thread-2) [n:127.0.0.1:34238_w_fob%2Fb ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42667/solr ready
[junit4] 2> 345589 INFO (closeThreadPool-2076-thread-2) [n:127.0.0.1:34238_w_fob%2Fb ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 345638 INFO (closeThreadPool-2076-thread-2) [n:127.0.0.1:34238_w_fob%2Fb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a
[junit4] 2> 345653 INFO (closeThreadPool-2076-thread-2) [n:127.0.0.1:34238_w_fob%2Fb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a
[junit4] 2> 345653 INFO (closeThreadPool-2076-thread-2) [n:127.0.0.1:34238_w_fob%2Fb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a
[junit4] 2> 345655 INFO (closeThreadPool-2076-thread-2) [n:127.0.0.1:34238_w_fob%2Fb ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.update.TestInPlaceUpdatesDistrib_5EB41C54616B95B7-001/shard-2-001/cores
[junit4] 2> 345696 WARN (closeThreadPool-2076-thread-1) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 345697 WARN (closeThreadPool-2076-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@1e72c922[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 345700 WARN (closeThreadPool-2076-thread-1) [ ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
[junit4] 2> 345701 WARN (closeThreadPool-2076-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@492982f3[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 345702 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42667/solr
[junit4] 2> 345704 INFO (zkConnectionManagerCallback-2120-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 345713 INFO (zkConnectionManagerCallback-2122-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 345717 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 345723 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.ZkController Publish node=127.0.0.1:34046_w_fob%2Fb as DOWN
[junit4] 2> 345725 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 345725 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34046_w_fob%2Fb
[junit4] 2> 345729 INFO (zkCallback-2063-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 345729 INFO (zkCallback-2105-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 345729 INFO (zkCallback-2074-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 345729 INFO (zkCallback-2092-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 345729 INFO (zkCallback-2085-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 345729 INFO (zkCallback-2056-thread-3) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 345730 INFO (zkCallback-2112-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 345730 INFO (zkCallback-2121-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 345758 INFO (zkConnectionManagerCallback-2129-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 345760 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 345761 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42667/solr ready
[junit4] 2> 345761 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 345790 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a
[junit4] 2> 345806 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a
[junit4] 2> 345807 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d2e4b2a
[junit4] 2> 345810 INFO (closeThreadPool-2076-thread-1) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/je
[...truncated too long message...]
_s=PostingsFormat(name=Asserting), 44parent_f2_s=Lucene50(blocksize=128), 34parent_f2_s=BlockTreeOrds(blocksize=128), 13parent_f2_s=PostingsFormat(name=Asserting), type_s=BlockTreeOrds(blocksize=128), 23parent_f2_s=BlockTreeOrds(blocksize=128), 43parent_f2_s=PostingsFormat(name=Direct), 5child1_s=PostingsFormat(name=Direct), 10parent_f1_s=Lucene50(blocksize=128), 33parent_f2_s=Lucene50(blocksize=128), 22parent_f2_s=Lucene50(blocksize=128), 4child0_s=PostingsFormat(name=Direct), 12parent_f2_s=BlockTreeOrds(blocksize=128), 42parent_f2_s=PostingsFormat(name=Asserting), 32parent_f2_s=PostingsFormat(name=Direct), 31parent_f1_s=PostingsFormat(name=Direct), 5grand1_s=BlockTreeOrds(blocksize=128), 6grand4_s=PostingsFormat(name=Direct), 11parent_f1_s=BlockTreeOrds(blocksize=128), 21parent_f1_s=Lucene50(blocksize=128), 4grand2_s=PostingsFormat(name=Direct), 7grand3_s=BlockTreeOrds(blocksize=128), 8grand2_s=PostingsFormat(name=Direct), 9grand5_s=PostingsFormat(name=Direct), 8grand6_s=PostingsFormat(name=Direct), 9grand1_s=PostingsFormat(name=Direct), 41parent_f1_s=PostingsFormat(name=Asserting), 37parent_f1_s=BlockTreeOrds(blocksize=128), 40parent_f2_s=Lucene50(blocksize=128), 47parent_f1_s=Lucene50(blocksize=128), 14parent_f1_s=Lucene50(blocksize=128), 20parent_f2_s=PostingsFormat(name=Asserting), 34parent_f1_s=PostingsFormat(name=Asserting), 24parent_f1_s=PostingsFormat(name=Direct), 44parent_f1_s=BlockTreeOrds(blocksize=128), 30parent_f2_s=BlockTreeOrds(blocksize=128), 5parent_f1_s=PostingsFormat(name=Asserting), 6grand0_s=PostingsFormat(name=Direct), 46parent_f2_s=PostingsFormat(name=Asserting), 10parent_f2_s=PostingsFormat(name=Direct), 5child0_s=Lucene50(blocksize=128), 8parent_f1_s=PostingsFormat(name=Direct), 27parent_f2_s=BlockTreeOrds(blocksize=128), 2parent_f2_s=PostingsFormat(name=Asserting), 4child1_s=Lucene50(blocksize=128), 2grand1_s=PostingsFormat(name=Asserting), 17parent_f2_s=PostingsFormat(name=Asserting), 3child2_s=Lucene50(blocksize=128), 16parent_f2_s=PostingsFormat(name=Direct), 26parent_f2_s=Lucene50(blocksize=128), 1child0_s=Lucene50(blocksize=128), 36parent_f2_s=PostingsFormat(name=Direct), 7grand6_s=Lucene50(blocksize=128), 18parent_f1_s=Lucene50(blocksize=128), 28parent_f1_s=BlockTreeOrds(blocksize=128), 5grand0_s=Lucene50(blocksize=128), 6grand5_s=PostingsFormat(name=Asserting), 7grand2_s=Lucene50(blocksize=128), 15parent_f1_s=BlockTreeOrds(blocksize=128), 4grand3_s=PostingsFormat(name=Asserting), 9grand8_s=Lucene50(blocksize=128), 8grand3_s=PostingsFormat(name=Asserting), 6parent_f1_s=BlockTreeOrds(blocksize=128), 25parent_f1_s=Lucene50(blocksize=128), 9grand4_s=Lucene50(blocksize=128), 8grand7_s=PostingsFormat(name=Asserting), 0parent_f1_s=PostingsFormat(name=Direct), 3grand2_s=Lucene50(blocksize=128), 35parent_f1_s=PostingsFormat(name=Direct), 9grand0_s=Lucene50(blocksize=128), 19parent_f2_s=Lucene50(blocksize=128), 4parent_f2_s=Lucene50(blocksize=128), 37parent_f2_s=Lucene50(blocksize=128), 47parent_f2_s=PostingsFormat(name=Direct), 5child4_s=Lucene50(blocksize=128), 38parent_f1_s=PostingsFormat(name=Asserting), 5grand4_s=Lucene50(blocksize=128), 6grand1_s=PostingsFormat(name=Asserting), 5parent_f2_s=PostingsFormat(name=Direct), 46parent_f1_s=PostingsFormat(name=Direct), 16parent_f1_s=PostingsFormat(name=Asserting), 8parent_f2_s=Lucene50(blocksize=128), 39parent_f2_s=PostingsFormat(name=Asserting), 29parent_f2_s=BlockTreeOrds(blocksize=128), 2parent_f1_s=BlockTreeOrds(blocksize=128), 4child2_s=BlockTreeOrds(blocksize=128), rnd_b=PostingsFormat(name=Asserting), 27parent_f1_s=PostingsFormat(name=Asserting), 45parent_f1_s=PostingsFormat(name=Asserting), 17parent_f1_s=BlockTreeOrds(blocksize=128), 2grand0_s=BlockTreeOrds(blocksize=128), 3child1_s=PostingsFormat(name=Direct), 7parent_f2_s=BlockTreeOrds(blocksize=128), 26parent_f1_s=BlockTreeOrds(blocksize=128), 2child0_s=PostingsFormat(name=Direct), 3parent_f1_s=Lucene50(blocksize=128), 36parent_f1_s=Lucene50(blocksize=128)}, docValues:{_version_=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=1796, maxMBSortInHeap=5.450108153089602, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@3bfdeb73), locale=is, timezone=Africa/Nouakchott
[junit4] 2> NOTE: Linux 4.4.0-137-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=225714224,total=478674944
[junit4] 2> NOTE: All tests run in this JVM: [TestImplicitCoreProperties, FileUtilsTest, TestUnifiedSolrHighlighter, CleanupOldIndexTest, FacetPivotSmallTest, DistributedQueueTest, BasicAuthIntegrationTest, TestQueryUtils, PathHierarchyTokenizerFactoryTest, LeaderElectionIntegrationTest, TestSolrCloudWithSecureImpersonation, BadIndexSchemaTest, TestSchemaResource, TestPayloadScoreQParserPlugin, TestOrdValues, DeleteReplicaTest, TestExactSharedStatsCache, TestLocalFSCloudBackupRestore, TestManagedResource, DistributedFacetPivotSmallTest, SystemInfoHandlerTest, DistributedDebugComponentTest, TestStressVersions, TestSolrQueryResponse, TestClusterProperties, TestJsonRequest, TestConfigsApi, TestReloadAndDeleteDocs, SuggesterFSTTest, TestPointFields, TestSkipOverseerOperations, TestManagedSchemaAPI, TestNestedDocsSort, CoreMergeIndexesAdminHandlerTest, MetricUtilsTest, BadComponentTest, TestInPlaceUpdatesDistrib, HighlighterMaxOffsetTest, TestSolrCloudSnapshots, SortSpecParsingTest, TestRealTimeGet, DOMUtilTest, BlockJoinFacetSimpleTest, TestSimDistributedQueue, TriggerEventQueueTest, TestTolerantUpdateProcessorRandomCloud, CloneFieldUpdateProcessorFactoryTest, TestFuzzyAnalyzedSuggestions, TestSolrCoreProperties, MergeStrategyTest, TestCrossCoreJoin, TestElisionMultitermQuery, DirectSolrConnectionTest, HdfsBasicDistributedZk2Test, JsonLoaderTest, TestDynamicFieldResource, SortByFunctionTest, CursorPagingTest, MetricTriggerTest, ForceLeaderTest, TestSha256AuthenticationProvider, TestSegmentSorting, TestMinHashQParser, TestSubQueryTransformer, TestConfig, MetricsHistoryHandlerTest, CollectionsAPIDistributedZkTest, SplitShardTest, ZkShardTermsTest, TestCloudManagedSchema, AnalysisAfterCoreReloadTest, TestPerFieldSimilarity, SolrIndexMetricsTest, TestSQLHandler, TestLeaderElectionZkExpiry, TestRawResponseWriter, SoftAutoCommitTest, DocValuesTest, SampleTest, TestDistributedStatsComponentCardinality, TestNamedUpdateProcessors, RestoreTriggerStateTest, IgnoreLargeDocumentProcessorFactoryTest, TestOnReconnectListenerSupport, NodeLostTriggerIntegrationTest, TestDocBasedVersionConstraints, TestFieldSortValues, PrimitiveFieldTypeTest, DeleteInactiveReplicaTest, CollectionReloadTest, TestZkChroot, TestDeprecatedFilters, PeerSyncTest, TestPullReplicaErrorHandling, AddReplicaTest, TestCollationFieldDocValues, ScheduledTriggerTest, DocExpirationUpdateProcessorFactoryTest, TestSimNodeLostTrigger, CachingDirectoryFactoryTest, TestJsonFacets, RankQueryTest, OverseerStatusTest, TestReplicationHandlerBackup, TestReload, TestBM25SimilarityFactory, SmileWriterTest, HttpPartitionOnCommitTest, ChaosMonkeySafeLeaderTest, TestFieldCacheSort, AdminHandlersProxyTest, PrimUtilsTest, TestExclusionRuleCollectionAccess, TestDistribDocBasedVersion, TermVectorComponentDistributedTest, NotRequiredUniqueKeyTest, CurrencyRangeFacetCloudTest, CoreAdminHandlerTest, TestSimGenericDistributedQueue, LukeRequestHandlerTest, TestSolr4Spatial, TestWordDelimiterFilterFactory, WrapperMergePolicyFactoryTest, PeerSyncWithLeaderAndIndexFingerprintCachingTest, TestDirectoryFactory, TestCloudRecovery, TestSearcherReuse, TestCollectionAPI, TestNumericRangeQuery64, TestCoreBackup, TestStressRecovery, LegacyCloudClusterPropTest, DistributedSpellCheckComponentTest, ResponseHeaderTest, PeerSyncReplicationTest, CSVRequestHandlerTest, SchemaVersionSpecificBehaviorTest, DisMaxRequestHandlerTest, DistributedIntervalFacetingTest, EchoParamsTest, MinimalSchemaTest, OutputWriterTest, TestDistributedMissingSort, TestGroupingSearch, TestRandomDVFaceting, ProtectedTermFilterFactoryTest, TestCharFilters, TokenizerChainTest, TestLuceneIndexBackCompat, TestEmbeddedSolrServerAdminHandler, TestJettySolrRunner, ConnectionReuseTest, ActionThrottleTest, AsyncCallRequestStatusResponseTest, BasicZkTest, CloudExitableDirectoryReaderTest, ClusterStateTest, ClusterStateUpdateTest, CollectionPropsTest, CollectionStateFormat2Test, CollectionsAPISolrJTest, ConcurrentCreateRoutedAliasTest, ConfigSetsAPITest, ConnectionManagerTest, CreateCollectionCleanupTest, CreateRoutedAliasTest, DeleteShardTest, DeleteStatusTest, DistribCursorPagingTest, DistribDocExpirationUpdateProcessorTest, DistribJoinFromCollectionTest, DistributedVersionInfoTest, DocValuesNotIndexedTest, FullSolrCloudDistribCmdsTest]
[junit4] Completed [838/838 (6!)] on J0 in 5202.53s, 1 test, 1 error <<< FAILURES!
[...truncated 1 lines...]
[junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20181221_214254_0692193261307122341339.sysout
[junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/heapdumps/java_pid21447.hprof ...
[junit4] Heap dump file created [673270992 bytes in 8.232 secs]
[junit4] <<< JVM J0: EOF ----
[...truncated 8912 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/build.xml:661: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-master/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid21447.hprof
* java_pid21457.hprof
Total time: 380 minutes 5 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any