You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/10/24 09:23:56 UTC
[JENKINS] Lucene-Solr-7.x-Linux (32bit/jdk1.8.0_144) - Build # 657
- Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/657/
Java: 32bit/jdk1.8.0_144 -client -XX:+UseConcMarkSweepGC
2 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestCloudPseudoReturnFields
Error Message:
There are still nodes recoverying - waited for 330 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
at __randomizedtesting.SeedInfo.seed([EB74A49AB2382530]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
at org.apache.solr.cloud.TestCloudPseudoReturnFields.waitForRecoveriesToFinish(TestCloudPseudoReturnFields.java:836)
at org.apache.solr.cloud.TestCloudPseudoReturnFields.createMiniSolrCloudCluster(TestCloudPseudoReturnFields.java:88)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth
Error Message:
2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth: 1) Thread[id=454, name=jetty-launcher-37-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323) at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105) at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41) at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244) at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44) at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61) at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) 2) Thread[id=450, name=jetty-launcher-37-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323) at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105) at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41) at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244) at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44) at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61) at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth:
1) Thread[id=454, name=jetty-launcher-37-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
2) Thread[id=450, name=jetty-launcher-37-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
at __randomizedtesting.SeedInfo.seed([EB74A49AB2382530]:0)
Build Log:
[...truncated 11604 lines...]
[junit4] Suite: org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth
[junit4] 2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_EB74A49AB2382530-001/init-core-data-001
[junit4] 2> 113489 WARN (SUITE-TestImpersonationWithHadoopAuth-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 113489 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 113491 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 113491 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 113523 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_EB74A49AB2382530-001/tempDir-001
[junit4] 2> 113523 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 113523 INFO (Thread-132) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 113523 INFO (Thread-132) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 113526 ERROR (Thread-132) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 113623 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:38503
[junit4] 2> 113765 INFO (jetty-launcher-37-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 113765 INFO (jetty-launcher-37-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 113800 INFO (jetty-launcher-37-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1627210{/solr,null,AVAILABLE}
[junit4] 2> 113800 INFO (jetty-launcher-37-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@e68dbe{/solr,null,AVAILABLE}
[junit4] 2> 113811 INFO (jetty-launcher-37-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1de314e{HTTP/1.1,[http/1.1]}{127.0.0.1:33679}
[junit4] 2> 113811 INFO (jetty-launcher-37-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4d8786{HTTP/1.1,[http/1.1]}{127.0.0.1:39815}
[junit4] 2> 113811 INFO (jetty-launcher-37-thread-1) [ ] o.e.j.s.Server Started @114868ms
[junit4] 2> 113811 INFO (jetty-launcher-37-thread-2) [ ] o.e.j.s.Server Started @114868ms
[junit4] 2> 113811 INFO (jetty-launcher-37-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39815}
[junit4] 2> 113811 INFO (jetty-launcher-37-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33679}
[junit4] 2> 113829 ERROR (jetty-launcher-37-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 113829 ERROR (jetty-launcher-37-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 113829 INFO (jetty-launcher-37-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 113829 INFO (jetty-launcher-37-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 113829 INFO (jetty-launcher-37-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 113829 INFO (jetty-launcher-37-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 113829 INFO (jetty-launcher-37-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 113829 INFO (jetty-launcher-37-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 113830 INFO (jetty-launcher-37-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-24T08:20:24.294Z
[junit4] 2> 113830 INFO (jetty-launcher-37-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-24T08:20:24.294Z
[junit4] 2> 113831 INFO (jetty-launcher-37-thread-1) [ ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
[junit4] 2> 113831 INFO (jetty-launcher-37-thread-2) [ ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
[junit4] 2> 113837 INFO (jetty-launcher-37-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 113844 INFO (jetty-launcher-37-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 113875 INFO (jetty-launcher-37-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38503/solr
[junit4] 2> 113875 INFO (jetty-launcher-37-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38503/solr
[junit4] 2> 113980 INFO (jetty-launcher-37-thread-2) [n:127.0.0.1:33679_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 113980 INFO (jetty-launcher-37-thread-1) [n:127.0.0.1:39815_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 113982 INFO (jetty-launcher-37-thread-2) [n:127.0.0.1:33679_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33679_solr
[junit4] 2> 113983 INFO (jetty-launcher-37-thread-1) [n:127.0.0.1:39815_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39815_solr
[junit4] 2> 113983 INFO (jetty-launcher-37-thread-2) [n:127.0.0.1:33679_solr ] o.a.s.c.Overseer Overseer (id=98882894167277574-127.0.0.1:33679_solr-n_0000000000) starting
[junit4] 2> 113993 INFO (zkCallback-48-thread-1-processing-n:127.0.0.1:33679_solr) [n:127.0.0.1:33679_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 113994 INFO (zkCallback-49-thread-1-processing-n:127.0.0.1:39815_solr) [n:127.0.0.1:39815_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 114071 INFO (jetty-launcher-37-thread-2) [n:127.0.0.1:33679_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33679_solr
[junit4] 2> 114072 INFO (zkCallback-49-thread-1-processing-n:127.0.0.1:39815_solr) [n:127.0.0.1:39815_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 114073 INFO (zkCallback-48-thread-1-processing-n:127.0.0.1:33679_solr) [n:127.0.0.1:33679_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 114166 INFO (jetty-launcher-37-thread-1) [n:127.0.0.1:39815_solr ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HadoopAuthPlugin
[junit4] 2> 114171 INFO (jetty-launcher-37-thread-1) [n:127.0.0.1:39815_solr ] o.a.s.s.HadoopAuthPlugin Params: {proxyuser.anyHostUsersGroup.hosts=*, proxyuser.wrongHost.hosts=1.1.1.1.1.1, proxyuser.bogusGroup.groups=__some_bogus_group, proxyuser.localHostAnyGroup.groups=*, cookie.domain=127.0.0.1, proxyuser.wrongHost.groups=*, type=simple, proxyuser.noHosts.groups=*, proxyuser.anyHostUsersGroup.groups=jenkins, proxyuser.noGroups.hosts=*, proxyuser.anyHostAnyUser.groups=*, delegation-token.token-kind=solr-dt, proxyuser.bogusGroup.hosts=*, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, zk-dt-secret-manager.enable=true, proxyuser.localHostAnyGroup.hosts=localhost.localdomain,localhost,127.0.0.1, proxyuser.anyHostAnyUser.hosts=*, signer.secret.provider=zookeeper}
[junit4] 2> 114219 INFO (jetty-launcher-37-thread-1) [n:127.0.0.1:39815_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 114230 INFO (jetty-launcher-37-thread-2) [n:127.0.0.1:33679_solr ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HadoopAuthPlugin
[junit4] 2> 114230 INFO (jetty-launcher-37-thread-2) [n:127.0.0.1:33679_solr ] o.a.s.s.HadoopAuthPlugin Params: {proxyuser.anyHostUsersGroup.hosts=*, proxyuser.wrongHost.hosts=1.1.1.1.1.1, proxyuser.bogusGroup.groups=__some_bogus_group, proxyuser.localHostAnyGroup.groups=*, cookie.domain=127.0.0.1, proxyuser.wrongHost.groups=*, type=simple, proxyuser.noHosts.groups=*, proxyuser.anyHostUsersGroup.groups=jenkins, proxyuser.noGroups.hosts=*, proxyuser.anyHostAnyUser.groups=*, delegation-token.token-kind=solr-dt, proxyuser.bogusGroup.hosts=*, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, zk-dt-secret-manager.enable=true, proxyuser.localHostAnyGroup.hosts=localhost.localdomain,localhost,127.0.0.1, proxyuser.anyHostAnyUser.hosts=*, signer.secret.provider=zookeeper}
[junit4] 2> 114238 INFO (jetty-launcher-37-thread-2) [n:127.0.0.1:33679_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 114247 INFO (jetty-launcher-37-thread-1-EventThread) [n:127.0.0.1:39815_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 114247 INFO (jetty-launcher-37-thread-2-EventThread) [n:127.0.0.1:33679_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 114332 INFO (jetty-launcher-37-thread-2) [n:127.0.0.1:33679_solr ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
[junit4] 2> 114332 INFO (jetty-launcher-37-thread-1) [n:127.0.0.1:39815_solr ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
[junit4] 2> 114411 INFO (jetty-launcher-37-thread-2) [n:127.0.0.1:33679_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33679.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c51e00
[junit4] 2> 114411 INFO (jetty-launcher-37-thread-1) [n:127.0.0.1:39815_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39815.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c51e00
[junit4] 2> 114412 INFO (jetty-launcher-37-thread-1) [n:127.0.0.1:39815_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39815.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c51e00
[junit4] 2> 114412 INFO (jetty-launcher-37-thread-2) [n:127.0.0.1:33679_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33679.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c51e00
[junit4] 2> 114412 INFO (jetty-launcher-37-thread-1) [n:127.0.0.1:39815_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39815.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c51e00
[junit4] 2> 114412 INFO (jetty-launcher-37-thread-2) [n:127.0.0.1:33679_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33679.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c51e00
[junit4] 2> 114415 INFO (jetty-launcher-37-thread-1) [n:127.0.0.1:39815_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_EB74A49AB2382530-001/tempDir-001/node1/.
[junit4] 2> 114415 INFO (jetty-launcher-37-thread-2) [n:127.0.0.1:33679_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_EB74A49AB2382530-001/tempDir-001/node2/.
[junit4] 2> 114522 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 114523 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38503/solr ready
[junit4] 2> 114535 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigGroups-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigGroups
[junit4] 2> 114754 WARN (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigGroups-seed#[EB74A49AB2382530]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noGroups&p=noGroups&t=simple-dt&e=1508869225174&s=k697U8ygbHL3iuF6EpuPfHzWElc="; Domain=127.0.0.1; Expires=?t, 24- ?j-2017 18:20:25 GMT; HttpOnly". Invalid 'expires' attribute: ?t, 24- ?j-2017 18:20:25 GMT
[junit4] 2> 114757 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigGroups-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigGroups
[junit4] 2> 114766 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyWrongHost-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyWrongHost
[junit4] 2> 129782 WARN (jetty-launcher-37-thread-1-SendThread(127.0.0.1:38503)) [n:127.0.0.1:39815_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15361ms for sessionid 0x15f4d77e5340005
[junit4] 2> 129783 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f4d77e5340005, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 129783 WARN (jetty-launcher-37-thread-1-SendThread(127.0.0.1:38503)) [n:127.0.0.1:39815_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15453ms for sessionid 0x15f4d77e5340007
[junit4] 2> 129783 WARN (jetty-launcher-37-thread-2-SendThread(127.0.0.1:38503)) [n:127.0.0.1:33679_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15362ms for sessionid 0x15f4d77e5340006
[junit4] 2> 129783 WARN (jetty-launcher-37-thread-2-SendThread(127.0.0.1:38503)) [n:127.0.0.1:33679_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15453ms for sessionid 0x15f4d77e5340008
[junit4] 2> 129784 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f4d77e5340007, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 129784 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f4d77e5340008, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 129784 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f4d77e5340006, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 129786 WARN (TEST-TestImpersonationWithHadoopAuth.testProxyWrongHost-seed#[EB74A49AB2382530]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=wrongHost&p=wrongHost&t=simple-dt&e=1508869225233&s=8+aSZ1sGBeCd0lmH4Hy0pQvDGNw="; Domain=127.0.0.1; Expires=?t, 24- ?j-2017 18:20:25 GMT; HttpOnly". Invalid 'expires' attribute: ?t, 24- ?j-2017 18:20:25 GMT
[junit4] 2> 129786 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyWrongHost-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyWrongHost
[junit4] 2> 129798 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyValidateHost-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateHost
[junit4] 2> 129812 INFO (qtp19655231-406) [n:127.0.0.1:39815_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&user.name=localHostAnyGroup&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 129813 INFO (qtp19655231-406) [n:127.0.0.1:39815_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&user.name=localHostAnyGroup&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 129817 WARN (TEST-TestImpersonationWithHadoopAuth.testProxyValidateHost-seed#[EB74A49AB2382530]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=simple-dt&e=1508869240265&s=b+pnYpq78RyoxnoB2T+iM7OMjKQ="; Domain=127.0.0.1; Expires=?t, 24- ?j-2017 18:20:40 GMT; HttpOnly". Invalid 'expires' attribute: ?t, 24- ?j-2017 18:20:40 GMT
[junit4] 2> 129817 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyValidateHost-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateHost
[junit4] 2> 129828 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyNullProxyUser-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNullProxyUser
[junit4] 2> 129832 INFO (qtp19655231-417) [ ] o.a.s.s.SolrDispatchFilter Error authenticating
[junit4] 2> java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
[junit4] 2> at org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler.authenticate(PseudoAuthenticationHandler.java:195)
[junit4] 2> at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
[junit4] 2> at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
[junit4] 2> at org.apache.solr.security.HadoopAuthFilter.doFilter(HadoopAuthFilter.java:110)
[junit4] 2> at org.apache.solr.security.HadoopAuthPlugin.doAuthenticate(HadoopAuthPlugin.java:245)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 129832 WARN (qtp19655231-417) [ ] o.e.j.s.ServletHandler /solr/admin/collections
[junit4] 2> org.apache.solr.common.SolrException: Error during request authentication,
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:457)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
[junit4] 2> at org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler.authenticate(PseudoAuthenticationHandler.java:195)
[junit4] 2> at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
[junit4] 2> at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
[junit4] 2> at org.apache.solr.security.HadoopAuthFilter.doFilter(HadoopAuthFilter.java:110)
[junit4] 2> at org.apache.solr.security.HadoopAuthPlugin.doAuthenticate(HadoopAuthPlugin.java:245)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
[junit4] 2> ... 26 more
[junit4] 2> 129836 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyNullProxyUser-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNullProxyUser
[junit4] 2> 129847 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyValidateGroup-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateGroup
[junit4] 2> 129850 INFO (qtp19655231-415) [n:127.0.0.1:39815_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=jenkins&action=LIST&user.name=anyHostUsersGroup&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 129850 INFO (qtp19655231-415) [n:127.0.0.1:39815_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=jenkins&action=LIST&user.name=anyHostUsersGroup&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 129851 WARN (TEST-TestImpersonationWithHadoopAuth.testProxyValidateGroup-seed#[EB74A49AB2382530]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostUsersGroup&p=anyHostUsersGroup&t=simple-dt&e=1508869240313&s=usTWGt0wAA9uGvSlSGDJa5he53E="; Domain=127.0.0.1; Expires=?t, 24- ?j-2017 18:20:40 GMT; HttpOnly". Invalid 'expires' attribute: ?t, 24- ?j-2017 18:20:40 GMT
[junit4] 2> 129851 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyValidateGroup-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateGroup
[junit4] 2> 129862 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidGroup-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidGroup
[junit4] 2> 129869 WARN (qtp19655231-403) [ ] o.a.h.s.ShellBasedUnixGroupsMapping got exception trying to get groups for user bar: id: ???bar???: no such user
[junit4] 2> id: ???bar???: no such user
[junit4] 2>
[junit4] 2> 129872 WARN (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidGroup-seed#[EB74A49AB2382530]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bogusGroup&p=bogusGroup&t=simple-dt&e=1508869240329&s=MW9ffFGzwuX06TCpMiJgLHBOFbo="; Domain=127.0.0.1; Expires=?t, 24- ?j-2017 18:20:40 GMT; HttpOnly". Invalid 'expires' attribute: ?t, 24- ?j-2017 18:20:40 GMT
[junit4] 2> 129873 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidGroup-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidGroup
[junit4] 2> 129883 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigHosts-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigHosts
[junit4] 2> 129884 INFO (jetty-launcher-37-thread-2-EventThread) [n:127.0.0.1:33679_solr ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
[junit4] 2> 129888 WARN (zkCallback-56-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@d5c137 name: ZooKeeperConnection Watcher:127.0.0.1:38503/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 129888 WARN (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigHosts-seed#[EB74A49AB2382530]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noHosts&p=noHosts&t=simple-dt&e=1508869240349&s=kh1foJQnYmD4tNvaLaGbwiPLXuA="; Domain=127.0.0.1; Expires=?t, 24- ?j-2017 18:20:40 GMT; HttpOnly". Invalid 'expires' attribute: ?t, 24- ?j-2017 18:20:40 GMT
[junit4] 2> 129888 WARN (zkCallback-56-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 129889 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigHosts-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigHosts
[junit4] 2> 129889 WARN (zkCallback-49-thread-2-processing-n:127.0.0.1:39815_solr) [n:127.0.0.1:39815_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@82f23d name: ZooKeeperConnection Watcher:127.0.0.1:38503/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 129890 WARN (zkCallback-49-thread-2-processing-n:127.0.0.1:39815_solr) [n:127.0.0.1:39815_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 129890 WARN (zkCallback-48-thread-2-processing-n:127.0.0.1:33679_solr) [n:127.0.0.1:33679_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@18f4750 name: ZooKeeperConnection Watcher:127.0.0.1:38503/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 129890 WARN (zkCallback-48-thread-2-processing-n:127.0.0.1:33679_solr) [n:127.0.0.1:33679_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 129906 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyValidateAnyHostAnyUser-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateAnyHostAnyUser
[junit4] 2> 129909 INFO (qtp19655231-406) [n:127.0.0.1:39815_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&user.name=anyHostAnyUser&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 129909 INFO (qtp19655231-406) [n:127.0.0.1:39815_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&user.name=anyHostAnyUser&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 129910 WARN (TEST-TestImpersonationWithHadoopAuth.testProxyValidateAnyHostAnyUser-seed#[EB74A49AB2382530]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostAnyUser&p=anyHostAnyUser&t=simple-dt&e=1508869240373&s=NLCEfnigAjqQHyQ3xrBdTJQbDZ0="; Domain=127.0.0.1; Expires=?t, 24- ?j-2017 18:20:40 GMT; HttpOnly". Invalid 'expires' attribute: ?t, 24- ?j-2017 18:20:40 GMT
[junit4] 2> 129910 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyValidateAnyHostAnyUser-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateAnyHostAnyUser
[junit4] 2> 129920 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidProxyUser-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidProxyUser
[junit4] 2> 129926 WARN (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidProxyUser-seed#[EB74A49AB2382530]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=simple-dt&e=1508869240387&s=F++ysIcdxc7LsnoUd89uOgJ5o/o="; Domain=127.0.0.1; Expires=?t, 24- ?j-2017 18:20:40 GMT; HttpOnly". Invalid 'expires' attribute: ?t, 24- ?j-2017 18:20:40 GMT
[junit4] 2> 129927 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidProxyUser-seed#[EB74A49AB2382530]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidProxyUser
[junit4] IGNOR/A 0.01s J2 | TestImpersonationWithHadoopAuth.testForwarding
[junit4] > Assumption #1: 'awaitsfix' test group is disabled (@AwaitsFix(bugUrl=https://issues.apache.org/jira/browse/HADOOP-9893))
[junit4] 2> 130492 INFO (jetty-closer-38-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@4d8786{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 130493 INFO (jetty-closer-38-thread-2) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@1de314e{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 130493 INFO (jetty-closer-38-thread-2) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=11582770
[junit4] 2> 130493 INFO (jetty-closer-38-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=14013600
[junit4] 2> 130493 INFO (jetty-closer-38-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 130493 INFO (jetty-closer-38-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 130493 INFO (jetty-closer-38-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1af20fd: rootName = solr_33679, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@1ac7239
[junit4] 2> 130493 INFO (jetty-closer-38-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@68baab: rootName = solr_39815, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@17a9846
[junit4] 2> 130503 INFO (jetty-closer-38-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 130503 INFO (jetty-closer-38-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@d0df6b: rootName = solr_33679, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@154ac33
[junit4] 2> 130504 INFO (jetty-closer-38-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 130504 INFO (jetty-closer-38-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@ded3a9: rootName = solr_39815, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@154ac33
[junit4] 2> 130515 INFO (jetty-closer-38-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 130515 INFO (jetty-closer-38-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@e7763b: rootName = solr_33679, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@e53f22
[junit4] 2> 130515 INFO (jetty-closer-38-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 130515 INFO (jetty-closer-38-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@15f938b: rootName = solr_39815, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@e53f22
[junit4] 2> 131051 INFO (jetty-closer-38-thread-2) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:33679_solr
[junit4] 2> 131052 INFO (jetty-closer-38-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 131052 INFO (jetty-closer-38-thread-2) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 131053 INFO (jetty-closer-38-thread-2) [ ] o.a.s.c.Overseer Overseer (id=98882894167277574-127.0.0.1:33679_solr-n_0000000000) closing
[junit4] 2> 131053 INFO (OverseerStateUpdate-98882894167277574-127.0.0.1:33679_solr-n_0000000000) [n:127.0.0.1:33679_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:33679_solr
[junit4] 2> 131054 WARN (OverseerAutoScalingTriggerThread-98882894167277574-127.0.0.1:33679_solr-n_0000000000) [n:127.0.0.1:33679_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 131054 INFO (OverseerCollectionConfigSetProcessor-98882894167277574-127.0.0.1:33679_solr-n_0000000000) [n:127.0.0.1:33679_solr ] o.a.s.c.OverseerTaskProcessor According to ZK I (id=98882894167277574-127.0.0.1:33679_solr-n_0000000000) am no longer a leader.
[junit4] 2> 131056 INFO (zkCallback-48-thread-6-processing-n:127.0.0.1:33679_solr) [n:127.0.0.1:33679_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 131056 ERROR (Thread[Thread-150,5,TGRP-TestImpersonationWithHadoopAuth]) [n:127.0.0.1:33679_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 131492 INFO (jetty-closer-38-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@e68dbe{/solr,null,UNAVAILABLE}
[junit4] 2> 131532 INFO (jetty-closer-38-thread-1) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:39815_solr
[junit4] 2> 131533 INFO (jetty-closer-38-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 131533 INFO (jetty-closer-38-thread-1) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 131534 INFO (zkCallback-49-thread-4-processing-n:127.0.0.1:39815_solr) [n:127.0.0.1:39815_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:39815_solr
[junit4] 2> 131534 INFO (zkCallback-49-thread-2-processing-n:127.0.0.1:39815_solr) [n:127.0.0.1:39815_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (0)
[junit4] 2> 131799 INFO (jetty-launcher-37-thread-1-EventThread) [n:127.0.0.1:39815_solr ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
[junit4] 2> 133034 ERROR (Thread[Thread-151,5,TGRP-TestImpersonationWithHadoopAuth]) [n:127.0.0.1:39815_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 133035 INFO (jetty-closer-38-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1627210{/solr,null,UNAVAILABLE}
[junit4] 2> 133036 ERROR (SUITE-TestImpersonationWithHadoopAuth-seed#[EB74A49AB2382530]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 133036 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38503 38503
[junit4] 2> 138070 INFO (Thread-132) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38503 38503
[junit4] 2> 138071 WARN (Thread-132) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/aliases.json
[junit4] 2> 3 /solr/clusterprops.json
[junit4] 2> 2 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
[junit4] 2> 3 /solr/clusterstate.json
[junit4] 2> 3 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/live_nodes
[junit4] 2> 3 /solr/collections
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
[junit4] 2>
[junit4] 2> 138071 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
[junit4] 2> Oct 24, 2017 8:20:48 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
[junit4] 2> Oct 24, 2017 8:21:08 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> SEVERE: 2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth:
[junit4] 2> 1) Thread[id=454, name=jetty-launcher-37-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
[junit4] 2> at sun.misc.Unsafe.park(Native Method)
[junit4] 2> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> 2) Thread[id=450, name=jetty-launcher-37-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
[junit4] 2> at sun.misc.Unsafe.park(Native Method)
[junit4] 2> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> Oct 24, 2017 8:21:08 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: Starting to interrupt leaked threads:
[junit4] 2> 1) Thread[id=454, name=jetty-launcher-37-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
[junit4] 2> 2) Thread[id=450, name=jetty-launcher-37-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
[junit4] 2> 158145 ERROR (jetty-launcher-37-thread-1-EventThread) [n:127.0.0.1:39815_solr ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
[junit4] 2> java.lang.InterruptedException
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> 158145 ERROR (jetty-launcher-37-thread-2-EventThread) [n:127.0.0.1:33679_solr ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
[junit4] 2> java.lang.InterruptedException
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> Oct 24, 2017 8:21:08 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: All leaked threads terminated.
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=700, maxMBSortInHeap=6.780771580879321, sim=RandomSimilarity(queryNorm=true): {}, locale=cs, timezone=America/Rosario
[junit4] 2> NOTE: Linux 4.10.0-33-generic i386/Oracle Corporation 1.8.0_144 (32-bit)/cpus=8,threads=1,free=21381784,total=67035136
[junit4] 2> NOTE: All tests run in this JVM: [DirectSolrSpellCheckerTest, UUIDFieldTest, PluginInfoTest, TestLFUCache, BlockJoinFacetSimpleTest, TestClassicSimilarityFactory, ZkStateReaderTest, HdfsDirectoryFactoryTest, SuggestComponentContextFilterQueryTest, TestLMJelinekMercerSimilarityFactory, TestImpersonationWithHadoopAuth]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestImpersonationWithHadoopAuth -Dtests.seed=EB74A49AB2382530 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=cs -Dtests.timezone=America/Rosario -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 0.00s J2 | TestImpersonationWithHadoopAuth (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth:
[junit4] > 1) Thread[id=454, name=jetty-launcher-37-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] > at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] > at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] > at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] > at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] > 2) Thread[id=450, name=jetty-launcher-37-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] > at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] > at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] > at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] > at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] > at __randomizedtesting.SeedInfo.seed([EB74A49AB2382530]:0)
[junit4] Completed [27/744 (1!)] on J2 in 44.68s, 10 tests, 1 error, 1 skipped <<< FAILURES!
[...truncated 362 lines...]
[junit4] Suite: org.apache.solr.cloud.TestCloudPseudoReturnFields
[junit4] 2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudPseudoReturnFields_EB74A49AB2382530-001/init-core-data-001
[junit4] 2> 732329 WARN (SUITE-TestCloudPseudoReturnFields-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=59 numCloses=59
[junit4] 2> 732329 INFO (SUITE-TestCloudPseudoReturnFields-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 732331 INFO (SUITE-TestCloudPseudoReturnFields-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 732332 INFO (SUITE-TestCloudPseudoReturnFields-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 732333 INFO (SUITE-TestCloudPseudoReturnFields-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudPseudoReturnFields_EB74A49AB2382530-001/tempDir-001
[junit4] 2> 732333 INFO (SUITE-TestCloudPseudoReturnFields-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 732333 INFO (Thread-735) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 732333 INFO (Thread-735) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 732345 ERROR (Thread-735) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 732433 INFO (SUITE-TestCloudPseudoReturnFields-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:45041
[junit4] 2> 732451 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f4d81566e0000, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 732456 INFO (jetty-launcher-716-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 732457 INFO (jetty-launcher-716-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 732459 INFO (jetty-launcher-716-thread-3) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 732460 INFO (jetty-launcher-716-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3275fa{/solr,null,AVAILABLE}
[junit4] 2> 732460 INFO (jetty-launcher-716-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@fa9a67{/solr,null,AVAILABLE}
[junit4] 2> 732460 INFO (jetty-launcher-716-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@190b817{/solr,null,AVAILABLE}
[junit4] 2> 732461 INFO (jetty-launcher-716-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1dd7778{SSL,[ssl, http/1.1]}{127.0.0.1:34123}
[junit4] 2> 732461 INFO (jetty-launcher-716-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1be5617{SSL,[ssl, http/1.1]}{127.0.0.1:42369}
[junit4] 2> 732461 INFO (jetty-launcher-716-thread-2) [ ] o.e.j.s.Server Started @733518ms
[junit4] 2> 732461 INFO (jetty-launcher-716-thread-3) [ ] o.e.j.s.Server Started @733518ms
[junit4] 2> 732461 INFO (jetty-launcher-716-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34123}
[junit4] 2> 732461 INFO (jetty-launcher-716-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=42369}
[junit4] 2> 732462 ERROR (jetty-launcher-716-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 732462 INFO (jetty-launcher-716-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@13f2d06{SSL,[ssl, http/1.1]}{127.0.0.1:34757}
[junit4] 2> 732462 INFO (jetty-launcher-716-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 732462 INFO (jetty-launcher-716-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 732462 INFO (jetty-launcher-716-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 732462 INFO (jetty-launcher-716-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-24T08:30:42.926Z
[junit4] 2> 732462 ERROR (jetty-launcher-716-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 732462 INFO (jetty-launcher-716-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 732462 INFO (jetty-launcher-716-thread-1) [ ] o.e.j.s.Server Started @733518ms
[junit4] 2> 732462 INFO (jetty-launcher-716-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 732462 INFO (jetty-launcher-716-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34757}
[junit4] 2> 732462 INFO (jetty-launcher-716-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 732462 INFO (jetty-launcher-716-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-24T08:30:42.926Z
[junit4] 2> 732463 ERROR (jetty-launcher-716-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 732463 INFO (jetty-launcher-716-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 732463 INFO (jetty-launcher-716-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 732463 INFO (jetty-launcher-716-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 732463 INFO (jetty-launcher-716-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-24T08:30:42.927Z
[junit4] 2> 732472 INFO (jetty-launcher-716-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 732494 INFO (jetty-launcher-716-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 732494 INFO (jetty-launcher-716-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 732528 INFO (jetty-launcher-716-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45041/solr
[junit4] 2> 732561 INFO (jetty-launcher-716-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45041/solr
[junit4] 2> 732576 INFO (jetty-launcher-716-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45041/solr
[junit4] 2> 732613 INFO (jetty-launcher-716-thread-2) [n:127.0.0.1:34123_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 732613 INFO (jetty-launcher-716-thread-3) [n:127.0.0.1:42369_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 732614 INFO (jetty-launcher-716-thread-2) [n:127.0.0.1:34123_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:34123_solr
[junit4] 2> 732614 INFO (jetty-launcher-716-thread-3) [n:127.0.0.1:42369_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42369_solr
[junit4] 2> 732615 INFO (jetty-launcher-716-thread-2) [n:127.0.0.1:34123_solr ] o.a.s.c.Overseer Overseer (id=98882934721609735-127.0.0.1:34123_solr-n_0000000000) starting
[junit4] 2> 732617 INFO (zkCallback-730-thread-1-processing-n:127.0.0.1:34123_solr) [n:127.0.0.1:34123_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 732630 INFO (zkCallback-734-thread-1-processing-n:127.0.0.1:42369_solr) [n:127.0.0.1:42369_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 732639 INFO (jetty-launcher-716-thread-2) [n:127.0.0.1:34123_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34123_solr
[junit4] 2> 732642 INFO (zkCallback-734-thread-1-processing-n:127.0.0.1:42369_solr) [n:127.0.0.1:42369_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 732651 INFO (zkCallback-730-thread-1-processing-n:127.0.0.1:34123_solr) [n:127.0.0.1:34123_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 732655 INFO (jetty-launcher-716-thread-1) [n:127.0.0.1:34757_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 732656 INFO (jetty-launcher-716-thread-1) [n:127.0.0.1:34757_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 732657 INFO (jetty-launcher-716-thread-1) [n:127.0.0.1:34757_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34757_solr
[junit4] 2> 732658 INFO (zkCallback-734-thread-1-processing-n:127.0.0.1:42369_solr) [n:127.0.0.1:42369_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 732678 INFO (zkCallback-725-thread-1-processing-n:127.0.0.1:34757_solr) [n:127.0.0.1:34757_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 732679 INFO (zkCallback-730-thread-2-processing-n:127.0.0.1:34123_solr) [n:127.0.0.1:34123_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 732826 INFO (jetty-launcher-716-thread-2) [n:127.0.0.1:34123_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34123.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c51e00
[junit4] 2> 732828 INFO (jetty-launcher-716-thread-1) [n:127.0.0.1:34757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34757.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c51e00
[junit4] 2> 732855 INFO (jetty-launcher-716-thread-1) [n:127.0.0.1:34757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34757.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c51e00
[junit4] 2> 732856 INFO (jetty-launcher-716-thread-1) [n:127.0.0.1:34757_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34757.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c51e00
[junit4] 2> 732857 INFO (jetty-launcher-716-thread-3) [n:127.0.0.1:42369_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42369.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c51e00
[junit4] 2> 732857 INFO (jetty-launcher-716-thread-1) [n:127.0.0.1:34757_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudPseudoReturnFields_EB74A49AB2382530-001/tempDir-001/node1/.
[junit4] 2> 732860 INFO (jetty-launcher-716-thread-2) [n:127.0.0.1:34123_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34123.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c51e00
[junit4] 2> 732860 INFO (jetty-launcher-716-thread-2) [n:127.0.0.1:34123_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34123.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c51e00
[junit4] 2> 732866 INFO (jetty-launcher-716-thread-2) [n:127.0.0.1:34123_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudPseudoReturnFields_EB74A49AB2382530-001/tempDir-001/node2/.
[junit4] 2> 732873 INFO (jetty-launcher-716-thread-3) [n:127.0.0.1:42369_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42369.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c51e00
[junit4] 2> 732873 INFO (jetty-launcher-716-thread-3) [n:127.0.0.1:42369_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42369.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@c51e00
[junit4] 2> 732883 INFO (jetty-launcher-716-thread-3) [n:127.0.0.1:42369_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudPseudoReturnFields_EB74A49AB2382530-001/tempDir-001/node3/.
[junit4] 2> 733000 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f4d81566e000a, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 733003 INFO (SUITE-TestCloudPseudoReturnFields-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 733004 INFO (SUITE-TestCloudPseudoReturnFields-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45041/solr ready
[junit4] 2> 733401 INFO (qtp19090443-3879) [n:127.0.0.1:34123_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&property.config=solrconfig-tlog.xml&property.schema=schema-psuedo-fields.xml&collection.configName=org.apache.solr.cloud.TestCloudPseudoReturnFields_config-set&name=org.apache.solr.cloud.TestCloudPseudoReturnFields_collection&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 733403 INFO (OverseerThreadFactory-1228-thread-1-processing-n:127.0.0.1:34123_solr) [n:127.0.0.1:34123_solr ] o.a.s.c.CreateCollectionCmd Create collection org.apache.solr.cloud.TestCloudPseudoReturnFields_collection
[junit4] 2> 733512 INFO (OverseerStateUpdate-98882934721609735-127.0.0.1:34123_solr-n_0000000000) [n:127.0.0.1:34123_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"org.apache.solr.cloud.TestCloudPseudoReturnFields_collection",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:42369/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 733516 INFO (OverseerStateUpdate-98882934721609735-127.0.0.1:34123_solr-n_0000000000) [n:127.0.0.1:34123_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"org.apache.solr.cloud.TestCloudPseudoReturnFields_collection",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard2_replica_n3",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:34757/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 733823 INFO (qtp8172954-3889) [n:127.0.0.1:42369_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=org.apache.solr.cloud.TestCloudPseudoReturnFields_config-set&newCollection=true&collection=org.apache.solr.cloud.TestCloudPseudoReturnFields_collection&version=2&replicaType=NRT&property.config=solrconfig-tlog.xml&property.schema=schema-psuedo-fields.xml&coreNodeName=core_node2&name=org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&wt=javabin
[junit4] 2> 733823 INFO (qtp8172954-3889) [n:127.0.0.1:42369_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 733841 INFO (qtp23972256-3888) [n:127.0.0.1:34757_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=org.apache.solr.cloud.TestCloudPseudoReturnFields_config-set&newCollection=true&collection=org.apache.solr.cloud.TestCloudPseudoReturnFields_collection&version=2&replicaType=NRT&property.config=solrconfig-tlog.xml&property.schema=schema-psuedo-fields.xml&coreNodeName=core_node4&name=org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard2_replica_n3&action=CREATE&numShards=2&shard=shard2&wt=javabin
[junit4] 2> 733841 INFO (qtp23972256-3888) [n:127.0.0.1:34757_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 733960 INFO (zkCallback-734-thread-1-processing-n:127.0.0.1:42369_solr) [n:127.0.0.1:42369_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudPseudoReturnFields_collection/state.json] for collection [org.apache.solr.cloud.TestCloudPseudoReturnFields_collection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 733961 INFO (zkCallback-734-thread-2-processing-n:127.0.0.1:42369_solr) [n:127.0.0.1:42369_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudPseudoReturnFields_collection/state.json] for collection [org.apache.solr.cloud.TestCloudPseudoReturnFields_collection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 733961 INFO (zkCallback-725-thread-1-processing-n:127.0.0.1:34757_solr) [n:127.0.0.1:34757_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudPseudoReturnFields_collection/state.json] for collection [org.apache.solr.cloud.TestCloudPseudoReturnFields_collection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 733961 INFO (zkCallback-725-thread-2-processing-n:127.0.0.1:34757_solr) [n:127.0.0.1:34757_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudPseudoReturnFields_collection/state.json] for collection [org.apache.solr.cloud.TestCloudPseudoReturnFields_collection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 734867 INFO (qtp8172954-3889) [n:127.0.0.1:42369_solr c:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 734897 INFO (qtp23972256-3888) [n:127.0.0.1:34757_solr c:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard2_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 734916 INFO (qtp8172954-3889) [n:127.0.0.1:42369_solr c:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard1_replica_n1] o.a.s.s.IndexSchema [org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 734959 INFO (qtp23972256-3888) [n:127.0.0.1:34757_solr c:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard2_replica_n3] o.a.s.s.IndexSchema [org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard2_replica_n3] Schema name=test
[junit4] 2> 734959 INFO (qtp8172954-3889) [n:127.0.0.1:42369_solr c:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.4 with uniqueid field id
[junit4] 2> 734985 INFO (qtp23972256-3888) [n:127.0.0.1:34757_solr c:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard2_replica_n3] o.a.s.s.IndexSchema Loaded schema test/1.4 with uniqueid field id
[junit4] 2> 734986 INFO (qtp8172954-3889) [n:127.0.0.1:42369_solr c:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard1_replica_n1' using configuration from collection org.apache.solr.cloud.TestCloudPseudoReturnFields_collection, trusted=true
[junit4] 2> 734987 INFO (qtp8172954-3889) [n:127.0.0.1:42369_solr c:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42369.solr.core.org.apache.solr.cloud.TestCloudPseudoReturnFields_collection.shar
[...truncated too long message...]
e.solr.cloud.TestCloudPseudoReturnFields_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard1_replica_n1] o.a.s.c.SolrCore [org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@edbe99
[junit4] 2> 1095814 INFO (coreCloseExecutor-1245-thread-1) [n:127.0.0.1:34757_solr c:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard2_replica_n3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.org.apache.solr.cloud.TestCloudPseudoReturnFields_collection.shard2.replica_n3, tag=3458623
[junit4] 2> 1095814 INFO (coreCloseExecutor-1246-thread-1) [n:127.0.0.1:42369_solr c:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.org.apache.solr.cloud.TestCloudPseudoReturnFields_collection.shard1.replica_n1, tag=15580825
[junit4] 2> 1095814 INFO (coreCloseExecutor-1245-thread-1) [n:127.0.0.1:34757_solr c:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard2_replica_n3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5bb9e1: rootName = solr_34757, domain = solr.core.org.apache.solr.cloud.TestCloudPseudoReturnFields_collection.shard2.replica_n3, service url = null, agent id = null] for registry solr.core.org.apache.solr.cloud.TestCloudPseudoReturnFields_collection.shard2.replica_n3 / com.codahale.metrics.MetricRegistry@173733
[junit4] 2> 1095814 INFO (coreCloseExecutor-1246-thread-1) [n:127.0.0.1:42369_solr c:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@bed4c5: rootName = solr_42369, domain = solr.core.org.apache.solr.cloud.TestCloudPseudoReturnFields_collection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.org.apache.solr.cloud.TestCloudPseudoReturnFields_collection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@15d57bc
[junit4] 2> 1095815 INFO (jetty-closer-717-thread-1) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:34123_solr
[junit4] 2> 1095815 INFO (jetty-closer-717-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 1095815 INFO (zkCallback-725-thread-3-processing-n:127.0.0.1:34757_solr) [n:127.0.0.1:34757_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 1095815 INFO (jetty-closer-717-thread-1) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1095816 INFO (zkCallback-734-thread-3-processing-n:127.0.0.1:42369_solr) [n:127.0.0.1:42369_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 1095818 INFO (zkCallback-730-thread-3-processing-n:127.0.0.1:34123_solr) [n:127.0.0.1:34123_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (1)
[junit4] 2> 1095820 INFO (jetty-closer-717-thread-1) [ ] o.a.s.c.Overseer Overseer (id=98882934721609735-127.0.0.1:34123_solr-n_0000000000) closing
[junit4] 2> 1095820 INFO (OverseerStateUpdate-98882934721609735-127.0.0.1:34123_solr-n_0000000000) [n:127.0.0.1:34123_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:34123_solr
[junit4] 2> 1095820 WARN (OverseerAutoScalingTriggerThread-98882934721609735-127.0.0.1:34123_solr-n_0000000000) [n:127.0.0.1:34123_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 1095831 INFO (zkCallback-730-thread-4-processing-n:127.0.0.1:34123_solr) [n:127.0.0.1:34123_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 1095831 INFO (jetty-closer-717-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3275fa{/solr,null,UNAVAILABLE}
[junit4] 2> 1095836 INFO (zkCallback-734-thread-3-processing-n:127.0.0.1:42369_solr) [n:127.0.0.1:42369_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:42369_solr
[junit4] 2> 1095859 INFO (coreCloseExecutor-1245-thread-1) [n:127.0.0.1:34757_solr c:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard2_replica_n3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.org.apache.solr.cloud.TestCloudPseudoReturnFields_collection.shard2.leader, tag=3458623
[junit4] 2> 1095862 INFO (coreCloseExecutor-1246-thread-1) [n:127.0.0.1:42369_solr c:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.org.apache.solr.cloud.TestCloudPseudoReturnFields_collection.shard1.leader, tag=15580825
[junit4] 2> 1095862 INFO (jetty-closer-717-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@fa9a67{/solr,null,UNAVAILABLE}
[junit4] 2> 1095865 INFO (jetty-closer-717-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@190b817{/solr,null,UNAVAILABLE}
[junit4] 2> 1095866 ERROR (SUITE-TestCloudPseudoReturnFields-seed#[EB74A49AB2382530]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1095867 INFO (SUITE-TestCloudPseudoReturnFields-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45041 45041
[junit4] 2> 1095896 INFO (Thread-735) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45041 45041
[junit4] 2> 1095897 WARN (Thread-735) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/aliases.json
[junit4] 2> 3 /solr/security.json
[junit4] 2> 2 /solr/configs/org.apache.solr.cloud.TestCloudPseudoReturnFields_config-set
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/collections/org.apache.solr.cloud.TestCloudPseudoReturnFields_collection/state.json
[junit4] 2> 4 /solr/clusterstate.json
[junit4] 2> 4 /solr/clusterprops.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/live_nodes
[junit4] 2> 4 /solr/collections
[junit4] 2>
[junit4] 2> 1095897 INFO (SUITE-TestCloudPseudoReturnFields-seed#[EB74A49AB2382530]-worker) [ ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
[junit4] 2> Oct 24, 2017 8:36:46 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 4 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1300, maxMBSortInHeap=6.588086334255592, sim=RandomSimilarity(queryNorm=true): {}, locale=mt-MT, timezone=Atlantic/St_Helena
[junit4] 2> NOTE: Linux 4.10.0-33-generic i386/Oracle Corporation 1.8.0_144 (32-bit)/cpus=8,threads=1,free=35657528,total=142479360
[junit4] 2> NOTE: All tests run in this JVM: [DirectSolrSpellCheckerTest, UUIDFieldTest, PluginInfoTest, TestLFUCache, BlockJoinFacetSimpleTest, TestClassicSimilarityFactory, ZkStateReaderTest, HdfsDirectoryFactoryTest, SuggestComponentContextFilterQueryTest, TestLMJelinekMercerSimilarityFactory, TestImpersonationWithHadoopAuth, TestMultiValuedNumericRangeQuery, TestGraphTermsQParserPlugin, BigEndianAscendingWordSerializerTest, TestTlogReplica, TestStreamBody, DocValuesNotIndexedTest, MetricsConfigTest, TestStressCloudBlindAtomicUpdates, SolrMetricReporterTest, TestSolrCloudWithDelegationTokens, TestRequestStatusCollectionAPI, BlockCacheTest, TestRebalanceLeaders, SolrSlf4jReporterTest, SolrCoreCheckLockOnStartupTest, DirectSolrConnectionTest, HdfsBasicDistributedZkTest, TestRandomCollapseQParserPlugin, TestCryptoKeys, SystemInfoHandlerTest, TestCloudPseudoReturnFields]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestCloudPseudoReturnFields -Dtests.seed=EB74A49AB2382530 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=mt-MT -Dtests.timezone=Atlantic/St_Helena -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 0.00s J2 | TestCloudPseudoReturnFields (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
[junit4] > at __randomizedtesting.SeedInfo.seed([EB74A49AB2382530]:0)
[junit4] > at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
[junit4] > at org.apache.solr.cloud.TestCloudPseudoReturnFields.waitForRecoveriesToFinish(TestCloudPseudoReturnFields.java:836)
[junit4] > at org.apache.solr.cloud.TestCloudPseudoReturnFields.createMiniSolrCloudCluster(TestCloudPseudoReturnFields.java:88)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] Completed [125/744 (2!)] on J2 in 364.36s, 0 tests, 1 failure <<< FAILURES!
[...truncated 45124 lines...]
[JENKINS] Lucene-Solr-7.x-Linux (32bit/jdk1.8.0_144) - Build # 660
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/660/
Java: 32bit/jdk1.8.0_144 -client -XX:+UseG1GC
7 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.analytics.legacy.facet.LegacyRangeFacetCloudTest
Error Message:
There are still nodes recoverying - waited for 90 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 90 seconds
at __randomizedtesting.SeedInfo.seed([17B944B40B529D96]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
at org.apache.solr.analytics.legacy.LegacyAbstractAnalyticsCloudTest.setupCollection(LegacyAbstractAnalyticsCloudTest.java:51)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.PeerSyncReplicationTest
Error Message:
20 threads leaked from SUITE scope at org.apache.solr.cloud.PeerSyncReplicationTest: 1) Thread[id=326, name=qtp12824006-326, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.waitForShardId(ZkController.java:1561) at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1512) at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1621) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1029) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:948) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91) at org.apache.solr.handler.admin.CoreAdminOperation$$Lambda$84/29221609.execute(Unknown Source) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) at org.eclipse.jetty.server.Server.handle(Server.java:534) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=363, name=zkCallback-70-thread-3-processing-n:127.0.0.1:44137_la_w, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:303) at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:551) at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:467) at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:454) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:788) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56) at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348) at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269) at org.apache.solr.common.cloud.SolrZkClient$3$$Lambda$172/33551235.run(Unknown Source) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$118/1353730.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=367, name=zkCallback-70-thread-6, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=339, name=Connection evictor, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=343, name=TEST-PeerSyncReplicationTest.test-seed#[FA782216D05FE784]-SendThread(127.0.0.1:33589), state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 6) Thread[id=366, name=zkCallback-70-thread-5, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=327, name=qtp12824006-327, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=328, name=qtp12824006-328, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=329, name=qtp12824006-329, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=332, name=qtp12824006-332, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=365, name=zkCallback-70-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=333, name=qtp12824006-333, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=334, name=org.eclipse.jetty.server.session.HashSessionManager@16f4803Timer, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=338, name=Connection evictor, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=344, name=TEST-PeerSyncReplicationTest.test-seed#[FA782216D05FE784]-EventThread, state=WAITING, group=TGRP-PeerSyncReplicationTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 16) Thread[id=349, name=Scheduler-1946079, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=346, name=Thread-71, state=WAITING, group=TGRP-PeerSyncReplicationTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 18) Thread[id=330, name=qtp12824006-330-acceptor-0@877827-ServerConnector@1a0edf2{SSL,[ssl, http/1.1]}{127.0.0.1:44137}, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=331, name=qtp12824006-331, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=368, name=zkCallback-70-thread-7, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 20 threads leaked from SUITE scope at org.apache.solr.cloud.PeerSyncReplicationTest:
1) Thread[id=326, name=qtp12824006-326, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.ZkController.waitForShardId(ZkController.java:1561)
at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1512)
at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1621)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1029)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:948)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
at org.apache.solr.handler.admin.CoreAdminOperation$$Lambda$84/29221609.execute(Unknown Source)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:534)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=363, name=zkCallback-70-thread-3-processing-n:127.0.0.1:44137_la_w, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:303)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:551)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:467)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:454)
at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:788)
at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
at org.apache.solr.common.cloud.SolrZkClient$3$$Lambda$172/33551235.run(Unknown Source)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$118/1353730.run(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=367, name=zkCallback-70-thread-6, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=339, name=Connection evictor, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=343, name=TEST-PeerSyncReplicationTest.test-seed#[FA782216D05FE784]-SendThread(127.0.0.1:33589), state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
6) Thread[id=366, name=zkCallback-70-thread-5, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=327, name=qtp12824006-327, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=328, name=qtp12824006-328, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=329, name=qtp12824006-329, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=332, name=qtp12824006-332, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=365, name=zkCallback-70-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=333, name=qtp12824006-333, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=334, name=org.eclipse.jetty.server.session.HashSessionManager@16f4803Timer, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=338, name=Connection evictor, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=344, name=TEST-PeerSyncReplicationTest.test-seed#[FA782216D05FE784]-EventThread, state=WAITING, group=TGRP-PeerSyncReplicationTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
16) Thread[id=349, name=Scheduler-1946079, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
17) Thread[id=346, name=Thread-71, state=WAITING, group=TGRP-PeerSyncReplicationTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678)
18) Thread[id=330, name=qtp12824006-330-acceptor-0@877827-ServerConnector@1a0edf2{SSL,[ssl, http/1.1]}{127.0.0.1:44137}, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
19) Thread[id=331, name=qtp12824006-331, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
20) Thread[id=368, name=zkCallback-70-thread-7, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([FA782216D05FE784]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.PeerSyncReplicationTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=326, name=qtp12824006-326, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=363, name=zkCallback-70-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=367, name=zkCallback-70-thread-6, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=343, name=TEST-PeerSyncReplicationTest.test-seed#[FA782216D05FE784]-SendThread(127.0.0.1:33589), state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 5) Thread[id=366, name=zkCallback-70-thread-5, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=327, name=qtp12824006-327, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=328, name=qtp12824006-328, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=329, name=qtp12824006-329, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest] at java.lang.Thread.isInterrupted(Native Method) at java.lang.Thread.isInterrupted(Thread.java:964) at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:218) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=365, name=zkCallback-70-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=334, name=org.eclipse.jetty.server.session.HashSessionManager@16f4803Timer, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=349, name=Scheduler-1946079, state=WAITING, group=TGRP-PeerSyncReplicationTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=346, name=Thread-71, state=WAITING, group=TGRP-PeerSyncReplicationTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 13) Thread[id=368, name=zkCallback-70-thread-7, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=326, name=qtp12824006-326, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
at sun.nio.ch.IOUtil.drain(Native Method)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=363, name=zkCallback-70-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=367, name=zkCallback-70-thread-6, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=343, name=TEST-PeerSyncReplicationTest.test-seed#[FA782216D05FE784]-SendThread(127.0.0.1:33589), state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
5) Thread[id=366, name=zkCallback-70-thread-5, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=327, name=qtp12824006-327, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
at sun.nio.ch.IOUtil.drain(Native Method)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=328, name=qtp12824006-328, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
at sun.nio.ch.IOUtil.drain(Native Method)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=329, name=qtp12824006-329, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
at java.lang.Thread.isInterrupted(Native Method)
at java.lang.Thread.isInterrupted(Thread.java:964)
at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:218)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=365, name=zkCallback-70-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=334, name=org.eclipse.jetty.server.session.HashSessionManager@16f4803Timer, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=349, name=Scheduler-1946079, state=WAITING, group=TGRP-PeerSyncReplicationTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=346, name=Thread-71, state=WAITING, group=TGRP-PeerSyncReplicationTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678)
13) Thread[id=368, name=zkCallback-70-thread-7, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([FA782216D05FE784]:0)
FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI
Error Message:
Error from server at http://127.0.0.1:33905/solr/awhollynewcollection_0: {"awhollynewcollection_0":5}
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:33905/solr/awhollynewcollection_0: {"awhollynewcollection_0":5}
at __randomizedtesting.SeedInfo.seed([FA782216D05FE784:B20D56A2D66CC811]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:972)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:972)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:972)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:972)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:972)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:460)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.PeerSyncReplicationTest.test
Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:36215/la_w
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:36215/la_w
at __randomizedtesting.SeedInfo.seed([FA782216D05FE784:722C1DCC7EA38A7C]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:637)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:418)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
at sun.security.ssl.InputRecord.read(InputRecord.java:503)
at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983)
at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:940)
at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
... 44 more
FAILED: org.apache.solr.cloud.SolrCloudExampleTest.testLoadDocsIntoGettingStartedCollection
Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:38571/oc_bdk
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:38571/oc_bdk
at __randomizedtesting.SeedInfo.seed([FA782216D05FE784:E91B1079E1305E22]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:637)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:315)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
at sun.security.ssl.InputRecord.read(InputRecord.java:503)
at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983)
at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:940)
at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
... 43 more
FAILED: org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory
Error Message:
replica never fully recovered
Stack Trace:
java.lang.AssertionError: replica never fully recovered
at __randomizedtesting.SeedInfo.seed([FA782216D05FE784:978486EB6A171883]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.waitForRecovery(AutoscalingHistoryHandlerTest.java:303)
at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:255)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImp
[...truncated too long message...]
/solr/build/contrib/solr-analytics/test/J0/temp/solr.analytics.legacy.facet.LegacyRangeFacetCloudTest_17B944B40B529D96-001/init-core-data-001
[junit4] 1> / (12)
[junit4] 1> /configs (1)
[junit4] 1> /configs/conf (2)
[junit4] 1> /configs/conf/schema.xml (0)
[junit4] 1> DATA: ...supressed...
[junit4] 1> /configs/conf/solrconfig.xml (0)
[junit4] 1> DATA: ...supressed...
[junit4] 1> /overseer (6)
[junit4] 1> DATA:
[junit4] 1>
[junit4] 1> /overseer/collection-queue-work (0)
[junit4] 1> /overseer/queue-work (0)
[junit4] 1> /overseer/collection-map-failure (0)
[junit4] 1> /overseer/collection-map-completed (0)
[junit4] 1> /overseer/queue (1)
[junit4] 1> /overseer/queue/qn-0000000009 (0)
[junit4] 1> DATA:
[junit4] 1> {
[junit4] 1> "core":"collection1_shard1_replica_n1",
[junit4] 1> "core_node_name":"core_node3",
[junit4] 1> "roles":null,
[junit4] 1> "base_url":"https://127.0.0.1:33063/solr",
[junit4] 1> "node_name":"127.0.0.1:33063_solr",
[junit4] 1> "numShards":"2",
[junit4] 1> "state":"active",
[junit4] 1> "shard":"shard1",
[junit4] 1> "collection":"collection1",
[junit4] 1> "type":"NRT",
[junit4] 1> "operation":"state"}
[junit4] 1> /overseer/collection-map-running (0)
[junit4] 1> /aliases.json (0)
[junit4] 1> /live_nodes (4)
[junit4] 1> /live_nodes/127.0.0.1:33063_solr (0)
[junit4] 1> /live_nodes/127.0.0.1:37219_solr (0)
[junit4] 1> /live_nodes/127.0.0.1:44385_solr (0)
[junit4] 1> /live_nodes/127.0.0.1:43541_solr (0)
[junit4] 1> /collections (1)
[junit4] 1> /collections/collection1 (4)
[junit4] 1> DATA:
[junit4] 1> {"configName":"conf"}
[junit4] 1> /collections/collection1/leaders (2)
[junit4] 1> /collections/collection1/leaders/shard2 (1)
[junit4] 1> /collections/collection1/leaders/shard2/leader (0)
[junit4] 1> DATA:
[junit4] 1> {
[junit4] 1> "core":"collection1_shard2_replica_n2",
[junit4] 1> "core_node_name":"core_node4",
[junit4] 1> "base_url":"https://127.0.0.1:43541/solr",
[junit4] 1> "node_name":"127.0.0.1:43541_solr"}
[junit4] 1> /collections/collection1/leaders/shard1 (1)
[junit4] 1> /collections/collection1/leaders/shard1/leader (0)
[junit4] 1> DATA:
[junit4] 1> {
[junit4] 1> "core":"collection1_shard1_replica_n1",
[junit4] 1> "core_node_name":"core_node3",
[junit4] 1> "base_url":"https://127.0.0.1:33063/solr",
[junit4] 1> "node_name":"127.0.0.1:33063_solr"}
[junit4] 1> /collections/collection1/counter (0)
[junit4] 1> DATA:
[junit4] 1>
[junit4] 1> /collections/collection1/state.json (0)
[junit4] 1> DATA:
[junit4] 1> {"collection1":{
[junit4] 1> "pullReplicas":"0",
[junit4] 1> "replicationFactor":"1",
[junit4] 1> "router":{"name":"compositeId"},
[junit4] 1> "maxShardsPerNode":"1",
[junit4] 1> "autoAddReplicas":"false",
[junit4] 1> "nrtReplicas":"1",
[junit4] 1> "tlogReplicas":"0",
[junit4] 1> "shards":{
[junit4] 1> "shard1":{
[junit4] 1> "range":"80000000-ffffffff",
[junit4] 1> "state":"active",
[junit4] 1> "replicas":{"core_node3":{
[junit4] 1> "core":"collection1_shard1_replica_n1",
[junit4] 1> "base_url":"https://127.0.0.1:33063/solr",
[junit4] 1> "node_name":"127.0.0.1:33063_solr",
[junit4] 1> "state":"down",
[junit4] 1> "type":"NRT",
[junit4] 1> "leader":"true"}}},
[junit4] 1> "shard2":{
[junit4] 1> "range":"0-7fffffff",
[junit4] 1> "state":"active",
[junit4] 1> "replicas":{"core_node4":{
[junit4] 1> "core":"collection1_shard2_replica_n2",
[junit4] 1> "base_url":"https://127.0.0.1:43541/solr",
[junit4] 1> "node_name":"127.0.0.1:43541_solr",
[junit4] 1> "state":"active",
[junit4] 1> "type":"NRT",
[junit4] 1> "leader":"true"}}}}}}
[junit4] 1> /collections/collection1/leader_elect (2)
[junit4] 1> /collections/collection1/leader_elect/shard2 (1)
[junit4] 1> /collections/collection1/leader_elect/shard2/election (1)
[junit4] 1> /collections/collection1/leader_elect/shard2/election/98885401931415564-core_node4-n_0000000000 (0)
[junit4] 1> /collections/collection1/leader_elect/shard1 (1)
[junit4] 1> /collections/collection1/leader_elect/shard1/election (1)
[junit4] 1> /collections/collection1/leader_elect/shard1/election/98885401931415562-core_node3-n_0000000000 (0)
[junit4] 1> /overseer_elect (2)
[junit4] 1> /overseer_elect/leader (0)
[junit4] 1> DATA:
[junit4] 1> {"id":"98885401931415562-127.0.0.1:33063_solr-n_0000000000"}
[junit4] 1> /overseer_elect/election (4)
[junit4] 1> /overseer_elect/election/98885401931415564-127.0.0.1:43541_solr-n_0000000002 (0)
[junit4] 1> /overseer_elect/election/98885401931415562-127.0.0.1:33063_solr-n_0000000000 (0)
[junit4] 1> /overseer_elect/election/98885401931415561-127.0.0.1:37219_solr-n_0000000001 (0)
[junit4] 1> /overseer_elect/election/98885401931415563-127.0.0.1:44385_solr-n_0000000003 (0)
[junit4] 1> /security.json (0)
[junit4] 1> DATA:
[junit4] 1> {}
[junit4] 1> /clusterstate.json (0)
[junit4] 1> DATA:
[junit4] 1> {}
[junit4] 1> /solr.xml (0)
[junit4] 1> DATA: ...supressed...
[junit4] 1> /autoscaling (4)
[junit4] 1> /autoscaling/nodeAdded (0)
[junit4] 1> /autoscaling/nodeLost (0)
[junit4] 1> /autoscaling/triggerState (1)
[junit4] 1> /autoscaling/triggerState/.auto_add_replicas (0)
[junit4] 1> DATA:
[junit4] 1> {
[junit4] 1> "lastLiveNodes":[
[junit4] 1> "127.0.0.1:33063_solr",
[junit4] 1> "127.0.0.1:37219_solr",
[junit4] 1> "127.0.0.1:43541_solr",
[junit4] 1> "127.0.0.1:44385_solr"],
[junit4] 1> "nodeNameVsTimeRemoved":{}}
[junit4] 1> /autoscaling/events (1)
[junit4] 1> /autoscaling/events/.auto_add_replicas (0)
[junit4] 1> /autoscaling.json (0)
[junit4] 1> DATA:
[junit4] 1> {
[junit4] 1> "cluster-preferences":[{
[junit4] 1> "minimize":"cores",
[junit4] 1> "precision":1}],
[junit4] 1> "triggers":{".auto_add_replicas":{
[junit4] 1> "name":".auto_add_replicas",
[junit4] 1> "event":"nodeLost",
[junit4] 1> "waitFor":30,
[junit4] 1> "actions":[
[junit4] 1> {
[junit4] 1> "name":"auto_add_replicas_plan",
[junit4] 1> "class":"solr.AutoAddReplicasPlanAction"},
[junit4] 1> {
[junit4] 1> "name":"execute_plan",
[junit4] 1> "class":"solr.ExecutePlanAction"}],
[junit4] 1> "enabled":true}},
[junit4] 1> "listeners":{".auto_add_replicas.system":{
[junit4] 1> "beforeAction":[],
[junit4] 1> "afterAction":[],
[junit4] 1> "trigger":".auto_add_replicas",
[junit4] 1> "stage":[
[junit4] 1> "WAITING",
[junit4] 1> "STARTED",
[junit4] 1> "ABORTED",
[junit4] 1> "SUCCEEDED",
[junit4] 1> "FAILED",
[junit4] 1> "BEFORE_ACTION",
[junit4] 1> "AFTER_ACTION"],
[junit4] 1> "class":"org.apache.solr.cloud.autoscaling.SystemLogListener"}}}
[junit4] 1> /clusterprops.json (0)
[junit4] 1> DATA:
[junit4] 1> {'urlScheme':'https'}
[junit4] 1>
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1102, maxMBSortInHeap=6.4170362157997145, sim=RandomSimilarity(queryNorm=false): {}, locale=el-CY, timezone=Europe/Budapest
[junit4] 2> NOTE: Linux 4.10.0-33-generic i386/Oracle Corporation 1.8.0_144 (32-bit)/cpus=8,threads=1,free=20628520,total=71303168
[junit4] 2> NOTE: All tests run in this JVM: [StringCastFunctionTest, LegacyRangeFacetCloudTest]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=LegacyRangeFacetCloudTest -Dtests.seed=17B944B40B529D96 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=el-CY -Dtests.timezone=Europe/Budapest -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J0 | LegacyRangeFacetCloudTest (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: There are still nodes recoverying - waited for 90 seconds
[junit4] > at __randomizedtesting.SeedInfo.seed([17B944B40B529D96]:0)
[junit4] > at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
[junit4] > at org.apache.solr.analytics.legacy.LegacyAbstractAnalyticsCloudTest.setupCollection(LegacyAbstractAnalyticsCloudTest.java:51)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] Completed [67/78 (1!)] on J0 in 129.16s, 0 tests, 1 failure <<< FAILURES!
[...truncated 40025 lines...]
[JENKINS] Lucene-Solr-7.x-Linux (64bit/jdk-9) - Build # 659 - Still
Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/659/
Java: 64bit/jdk-9 -XX:-UseCompressedOops -XX:+UseG1GC --illegal-access=deny
6 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.analytics.legacy.facet.LegacyFieldFacetCloudTest
Error Message:
Error from server at http://127.0.0.1:44781/solr: Could not fully create collection: collection1
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44781/solr: Could not fully create collection: collection1
at __randomizedtesting.SeedInfo.seed([144D2689CA067CE2]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.analytics.legacy.LegacyAbstractAnalyticsCloudTest.setupCollection(LegacyAbstractAnalyticsCloudTest.java:50)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:844)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
Error Message:
9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 1) Thread[id=22059, name=zkCallback-5427-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 2) Thread[id=22057, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B3D925FE12D9319C]-SendThread(127.0.0.1:32793), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.base@9/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 3) Thread[id=22193, name=zkCallback-5427-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 4) Thread[id=22260, name=zkCallback-5427-thread-6, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 5) Thread[id=22194, name=zkCallback-5427-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 6) Thread[id=22056, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.base@9/java.lang.Thread.sleep(Native Method) at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.base@9/java.lang.Thread.run(Thread.java:844) 7) Thread[id=22192, name=zkCallback-5427-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 8) Thread[id=22223, name=zkCallback-5427-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 9) Thread[id=22058, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B3D925FE12D9319C]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062) at java.base@9/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:
1) Thread[id=22059, name=zkCallback-5427-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
at java.base@9/java.lang.Thread.run(Thread.java:844)
2) Thread[id=22057, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B3D925FE12D9319C]-SendThread(127.0.0.1:32793), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.base@9/java.lang.Thread.sleep(Native Method)
at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
3) Thread[id=22193, name=zkCallback-5427-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
at java.base@9/java.lang.Thread.run(Thread.java:844)
4) Thread[id=22260, name=zkCallback-5427-thread-6, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
at java.base@9/java.lang.Thread.run(Thread.java:844)
5) Thread[id=22194, name=zkCallback-5427-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
at java.base@9/java.lang.Thread.run(Thread.java:844)
6) Thread[id=22056, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.base@9/java.lang.Thread.sleep(Native Method)
at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.base@9/java.lang.Thread.run(Thread.java:844)
7) Thread[id=22192, name=zkCallback-5427-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
at java.base@9/java.lang.Thread.run(Thread.java:844)
8) Thread[id=22223, name=zkCallback-5427-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
at java.base@9/java.lang.Thread.run(Thread.java:844)
9) Thread[id=22058, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B3D925FE12D9319C]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@9/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
at java.base@9/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2062)
at java.base@9/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
at __randomizedtesting.SeedInfo.seed([B3D925FE12D9319C]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=22059, name=zkCallback-5427-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 2) Thread[id=22057, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B3D925FE12D9319C]-SendThread(127.0.0.1:32793), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.base@9/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 3) Thread[id=22193, name=zkCallback-5427-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 4) Thread[id=22260, name=zkCallback-5427-thread-6, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 5) Thread[id=22194, name=zkCallback-5427-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 6) Thread[id=22192, name=zkCallback-5427-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844) 7) Thread[id=22223, name=zkCallback-5427-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.base@9/jdk.internal.misc.Unsafe.park(Native Method) at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091) at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152) at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641) at java.base@9/java.lang.Thread.run(Thread.java:844)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=22059, name=zkCallback-5427-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
at java.base@9/java.lang.Thread.run(Thread.java:844)
2) Thread[id=22057, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B3D925FE12D9319C]-SendThread(127.0.0.1:32793), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.base@9/java.lang.Thread.sleep(Native Method)
at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
3) Thread[id=22193, name=zkCallback-5427-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
at java.base@9/java.lang.Thread.run(Thread.java:844)
4) Thread[id=22260, name=zkCallback-5427-thread-6, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
at java.base@9/java.lang.Thread.run(Thread.java:844)
5) Thread[id=22194, name=zkCallback-5427-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
at java.base@9/java.lang.Thread.run(Thread.java:844)
6) Thread[id=22192, name=zkCallback-5427-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
at java.base@9/java.lang.Thread.run(Thread.java:844)
7) Thread[id=22223, name=zkCallback-5427-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
at java.base@9/java.lang.Thread.run(Thread.java:844)
at __randomizedtesting.SeedInfo.seed([B3D925FE12D9319C]:0)
FAILED: org.apache.solr.cloud.OverseerTest.testShardLeaderChange
Error Message:
Unexpected shard leader coll:collection1 shard:shard1 expected:<core[4]> but was:<core[1]>
Stack Trace:
org.junit.ComparisonFailure: Unexpected shard leader coll:collection1 shard:shard1 expected:<core[4]> but was:<core[1]>
at __randomizedtesting.SeedInfo.seed([B3D925FE12D9319C:6D8AA2090841C46D]:0)
at org.junit.Assert.assertEquals(Assert.java:125)
at org.apache.solr.cloud.OverseerTest.verifyShardLeader(OverseerTest.java:564)
at org.apache.solr.cloud.OverseerTest.testShardLeaderChange(OverseerTest.java:861)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:844)
FAILED: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test
Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:40645/julk/t/collection1
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:40645/julk/t/collection1
at __randomizedtesting.SeedInfo.seed([B3D925FE12D9319C:3B8D1A24BC255C64]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:637)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1583)
at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:213)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:844)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.base/java.net.SocketInputStream.socketRead0(Native Method)
at java.base/java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.base/java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.base/java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
... 52 more
FAILED: org.apache.solr.cloud.autoscaling.ExecutePlanActionTest.testIntegration
Error Message:
Timed out waiting for replicas of collection to be 2 again null Live Nodes: [127.0.0.1:40419_solr] Last available state: DocCollection(testIntegration//collections/testIntegration/state.json/6)={ "pullReplicas":"0", "replicationFactor":"2", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{ "core_node3":{ "core":"testIntegration_shard1_replica_n1", "base_url":"http://127.0.0.1:40419/solr", "node_name":"127.0.0.1:40419_solr", "state":"active", "type":"NRT", "leader":"true"}, "core_node4":{ "core":"testIntegration_shard1_replica_n2", "base_url":"http://127.0.0.1:33627/solr", "node_name":"127.0.0.1:33627_solr", "state":"down", "type":"NRT"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"2", "tlogReplicas":"0"}
Stack Trace:
java.lang.AssertionError: Timed out waiting for replicas of collection to be 2 again
null
Live Nodes: [127.0.0.1:40419_solr]
Last available state: DocCollection(testIntegration//collections/testIntegration/state.json/6)={
"pullReplicas":"0",
"replicationFactor":"2",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{
"core_node3":{
"core":"testIntegration_shard1_replica_n1",
"base_url":"http://127.0.0.1:40419/solr",
"node_name":"127.0.0.1:40419_solr",
"state":"active",
"type":"NRT",
"leader":"true"},
"core_node4":{
"core":"testIntegration_shard1_replica_n2",
"base_url":"http://127.0.0.1:33627/solr",
"node_name":"127.0.0.1:33627_solr",
"state":"down",
"type":"NRT"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"2",
"tlogReplicas":"0"}
at __randomizedtesting.SeedInfo.seed([B3D925FE12D9319C:3B82BD237E690B9]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
at org.apache.solr.cloud.autoscaling.ExecutePlanActionTest.testIntegration(ExecutePlanActionTest.java:209)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:844)
Build Log:
[...truncated 12742 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.ExecutePlanActionTest
[junit4] 2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ExecutePlanActionTest_B3D925FE12D9319C-001/init-core-data-001
[junit4] 2> 957296 WARN (SUITE-ExecutePlanActionTest-seed#[B3D925FE12D9319C]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=45 numCloses=45
[junit4] 2> 957297 INFO (SUITE-ExecutePlanActionTest-seed#[B3D925FE12D9319C]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 957297 INFO (SUITE-ExecutePlanActionTest-seed#[B3D925FE12D9319C]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, clientAuth=0.0/0.0)
[junit4] 2> 957297 INFO (SUITE-ExecutePlanActionTest-seed#[B3D925FE12D9319C]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 957298 INFO (SUITE-ExecutePlanActionTest-seed#[B3D925FE12D9319C]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ExecutePlanActionTest_B3D925FE12D9319C-001/tempDir-001
[junit4] 2> 957298 INFO (SUITE-ExecutePlanActionTest-seed#[B3D925FE12D9319C]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 957298 INFO (Thread-2118) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 957298 INFO (Thread-2118) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 957310 ERROR (Thread-2118) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 957398 INFO (SUITE-ExecutePlanActionTest-seed#[B3D925FE12D9319C]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:37407
[junit4] 2> 957402 INFO (jetty-launcher-4761-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 957410 INFO (jetty-launcher-4761-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 957422 INFO (jetty-launcher-4761-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@abfbac7{/solr,null,AVAILABLE}
[junit4] 2> 957425 INFO (jetty-launcher-4761-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@104fe63d{/solr,null,AVAILABLE}
[junit4] 2> 957430 INFO (jetty-launcher-4761-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7dc74e21{HTTP/1.1,[http/1.1]}{127.0.0.1:40419}
[junit4] 2> 957430 INFO (jetty-launcher-4761-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@577ade6c{HTTP/1.1,[http/1.1]}{127.0.0.1:33627}
[junit4] 2> 957430 INFO (jetty-launcher-4761-thread-1) [ ] o.e.j.s.Server Started @959101ms
[junit4] 2> 957430 INFO (jetty-launcher-4761-thread-2) [ ] o.e.j.s.Server Started @959101ms
[junit4] 2> 957430 INFO (jetty-launcher-4761-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40419}
[junit4] 2> 957430 INFO (jetty-launcher-4761-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33627}
[junit4] 2> 957431 ERROR (jetty-launcher-4761-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 957431 ERROR (jetty-launcher-4761-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 957431 INFO (jetty-launcher-4761-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 957431 INFO (jetty-launcher-4761-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 957431 INFO (jetty-launcher-4761-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 957431 INFO (jetty-launcher-4761-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 957431 INFO (jetty-launcher-4761-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 957431 INFO (jetty-launcher-4761-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 957431 INFO (jetty-launcher-4761-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-24T14:39:21.067265Z
[junit4] 2> 957431 INFO (jetty-launcher-4761-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-24T14:39:21.067288Z
[junit4] 2> 957443 INFO (jetty-launcher-4761-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 957443 INFO (jetty-launcher-4761-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 957453 INFO (jetty-launcher-4761-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37407/solr
[junit4] 2> 957453 INFO (jetty-launcher-4761-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37407/solr
[junit4] 2> 957468 INFO (jetty-launcher-4761-thread-2) [ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 957468 INFO (jetty-launcher-4761-thread-2) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33627_solr
[junit4] 2> 957470 INFO (jetty-launcher-4761-thread-2) [ ] o.a.s.c.Overseer Overseer (id=98884384255901702-127.0.0.1:33627_solr-n_0000000000) starting
[junit4] 2> 957530 INFO (jetty-launcher-4761-thread-2) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33627_solr
[junit4] 2> 957531 DEBUG (OverseerAutoScalingTriggerThread-98884384255901702-127.0.0.1:33627_solr-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 957555 DEBUG (OverseerAutoScalingTriggerThread-98884384255901702-127.0.0.1:33627_solr-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 957556 DEBUG (OverseerAutoScalingTriggerThread-98884384255901702-127.0.0.1:33627_solr-n_0000000000) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: []
[junit4] 2> 957556 DEBUG (OverseerAutoScalingTriggerThread-98884384255901702-127.0.0.1:33627_solr-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
[junit4] 2> 957556 DEBUG (OverseerAutoScalingTriggerThread-98884384255901702-127.0.0.1:33627_solr-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 957562 INFO (zkCallback-4773-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 957571 INFO (zkCallback-4772-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 957571 INFO (jetty-launcher-4761-thread-1) [ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 957572 INFO (jetty-launcher-4761-thread-1) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40419_solr
[junit4] 2> 957577 DEBUG (OverseerAutoScalingTriggerThread-98884384255901702-127.0.0.1:33627_solr-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 957578 DEBUG (OverseerAutoScalingTriggerThread-98884384255901702-127.0.0.1:33627_solr-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 957578 DEBUG (ScheduledTrigger-4143-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:33627_solr]
[junit4] 2> 957581 INFO (zkCallback-4773-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 957591 INFO (jetty-launcher-4761-thread-2) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33627.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbf6221
[junit4] 2> 957594 INFO (zkCallback-4772-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 957596 INFO (jetty-launcher-4761-thread-2) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33627.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbf6221
[junit4] 2> 957596 INFO (jetty-launcher-4761-thread-2) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33627.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbf6221
[junit4] 2> 957597 INFO (jetty-launcher-4761-thread-2) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ExecutePlanActionTest_B3D925FE12D9319C-001/tempDir-001/node2/.
[junit4] 2> 957619 INFO (jetty-launcher-4761-thread-1) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40419.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbf6221
[junit4] 2> 957626 INFO (jetty-launcher-4761-thread-1) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40419.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbf6221
[junit4] 2> 957626 INFO (jetty-launcher-4761-thread-1) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40419.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbf6221
[junit4] 2> 957628 INFO (jetty-launcher-4761-thread-1) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ExecutePlanActionTest_B3D925FE12D9319C-001/tempDir-001/node1/.
[junit4] 2> 957677 INFO (SUITE-ExecutePlanActionTest-seed#[B3D925FE12D9319C]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 957677 INFO (SUITE-ExecutePlanActionTest-seed#[B3D925FE12D9319C]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37407/solr ready
[junit4] 2> 957686 INFO (TEST-ExecutePlanActionTest.testExecute-seed#[B3D925FE12D9319C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testExecute
[junit4] 2> 957687 DEBUG (zkCallback-4773-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 957688 DEBUG (OverseerAutoScalingTriggerThread-98884384255901702-127.0.0.1:33627_solr-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 957691 DEBUG (OverseerAutoScalingTriggerThread-98884384255901702-127.0.0.1:33627_solr-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 957692 DEBUG (OverseerAutoScalingTriggerThread-98884384255901702-127.0.0.1:33627_solr-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 957692 DEBUG (OverseerAutoScalingTriggerThread-98884384255901702-127.0.0.1:33627_solr-n_0000000000) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 957694 INFO (qtp160687007-17879) [ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=testExecute&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 957696 INFO (OverseerThreadFactory-4145-thread-1) [ ] o.a.s.c.CreateCollectionCmd Create collection testExecute
[junit4] 2> 957806 INFO (OverseerStateUpdate-98884384255901702-127.0.0.1:33627_solr-n_0000000000) [ ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testExecute",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"testExecute_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:33627/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 957812 INFO (OverseerStateUpdate-98884384255901702-127.0.0.1:33627_solr-n_0000000000) [ ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testExecute",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"testExecute_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:40419/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 958027 INFO (qtp160687007-17883) [ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=testExecute_shard1_replica_n2&action=CREATE&numShards=1&collection=testExecute&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 958027 INFO (qtp160687007-17883) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 958039 INFO (qtp401615291-17886) [ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=testExecute_shard1_replica_n1&action=CREATE&numShards=1&collection=testExecute&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 958039 INFO (qtp401615291-17886) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 958142 INFO (zkCallback-4773-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 958142 INFO (zkCallback-4773-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 958142 INFO (zkCallback-4772-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 958142 INFO (zkCallback-4772-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 959037 INFO (qtp160687007-17883) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 959044 INFO (qtp160687007-17883) [ ] o.a.s.s.IndexSchema [testExecute_shard1_replica_n2] Schema name=minimal
[junit4] 2> 959046 INFO (qtp160687007-17883) [ ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 959046 INFO (qtp160687007-17883) [ ] o.a.s.c.CoreContainer Creating SolrCore 'testExecute_shard1_replica_n2' using configuration from collection testExecute, trusted=true
[junit4] 2> 959046 INFO (qtp160687007-17883) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40419.solr.core.testExecute.shard1.replica_n2' (registry 'solr.core.testExecute.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbf6221
[junit4] 2> 959047 INFO (qtp160687007-17883) [ ] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 959047 INFO (qtp160687007-17883) [ ] o.a.s.c.SolrCore [[testExecute_shard1_replica_n2] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ExecutePlanActionTest_B3D925FE12D9319C-001/tempDir-001/node1/testExecute_shard1_replica_n2], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ExecutePlanActionTest_B3D925FE12D9319C-001/tempDir-001/node1/./testExecute_shard1_replica_n2/data/]
[junit4] 2> 959049 INFO (qtp401615291-17886) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 959055 INFO (qtp401615291-17886) [ ] o.a.s.s.IndexSchema [testExecute_shard1_replica_n1] Schema name=minimal
[junit4] 2> 959057 INFO (qtp401615291-17886) [ ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 959057 INFO (qtp401615291-17886) [ ] o.a.s.c.CoreContainer Creating SolrCore 'testExecute_shard1_replica_n1' using configuration from collection testExecute, trusted=true
[junit4] 2> 959057 INFO (qtp401615291-17886) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33627.solr.core.testExecute.shard1.replica_n1' (registry 'solr.core.testExecute.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbf6221
[junit4] 2> 959057 INFO (qtp401615291-17886) [ ] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 959057 INFO (qtp401615291-17886) [ ] o.a.s.c.SolrCore [[testExecute_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ExecutePlanActionTest_B3D925FE12D9319C-001/tempDir-001/node2/testExecute_shard1_replica_n1], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ExecutePlanActionTest_B3D925FE12D9319C-001/tempDir-001/node2/./testExecute_shard1_replica_n1/data/]
[junit4] 2> 959089 INFO (qtp160687007-17883) [ ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 959089 INFO (qtp160687007-17883) [ ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 959096 INFO (qtp160687007-17883) [ ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 959096 INFO (qtp160687007-17883) [ ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 959097 INFO (qtp160687007-17883) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@42653126[testExecute_shard1_replica_n2] main]
[junit4] 2> 959098 INFO (qtp160687007-17883) [ ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 959098 INFO (qtp160687007-17883) [ ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 959099 INFO (qtp160687007-17883) [ ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 959099 INFO (qtp401615291-17886) [ ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 959099 INFO (qtp401615291-17886) [ ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 959099 INFO (searcherExecutor-4150-thread-1) [ ] o.a.s.c.SolrCore [testExecute_shard1_replica_n2] Registered new searcher Searcher@42653126[testExecute_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 959100 INFO (qtp160687007-17883) [ ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582150149981863936
[junit4] 2> 959101 INFO (qtp401615291-17886) [ ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 959101 INFO (qtp401615291-17886) [ ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 959102 INFO (qtp401615291-17886) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@1e36440e[testExecute_shard1_replica_n1] main]
[junit4] 2> 959102 INFO (qtp401615291-17886) [ ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 959103 INFO (qtp401615291-17886) [ ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 959103 INFO (qtp160687007-17883) [ ] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 959103 INFO (qtp401615291-17886) [ ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 959104 INFO (searcherExecutor-4151-thread-1) [ ] o.a.s.c.SolrCore [testExecute_shard1_replica_n1] Registered new searcher Searcher@1e36440e[testExecute_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 959104 INFO (qtp401615291-17886) [ ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582150149986058240
[junit4] 2> 959204 INFO (zkCallback-4773-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 959204 INFO (zkCallback-4772-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 959204 INFO (zkCallback-4772-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 959204 INFO (zkCallback-4773-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 959604 INFO (qtp160687007-17883) [ ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 959604 INFO (qtp160687007-17883) [ ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 959604 INFO (qtp160687007-17883) [ ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:40419/solr/testExecute_shard1_replica_n2/
[junit4] 2> 959604 INFO (qtp160687007-17883) [ ] o.a.s.u.PeerSync PeerSync: core=testExecute_shard1_replica_n2 url=http://127.0.0.1:40419/solr START replicas=[http://127.0.0.1:33627/solr/testExecute_shard1_replica_n1/] nUpdates=100
[junit4] 2> 959606 INFO (qtp401615291-17887) [ ] o.a.s.c.S.Request [testExecute_shard1_replica_n1] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 959905 INFO (qtp160687007-17883) [ ] o.a.s.u.PeerSync PeerSync: core=testExecute_shard1_replica_n2 url=http://127.0.0.1:40419/solr DONE. We have no versions. sync failed.
[junit4] 2> 959905 INFO (qtp160687007-17883) [ ] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 959905 INFO (qtp160687007-17883) [ ] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 959905 INFO (qtp160687007-17883) [ ] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 959906 INFO (qtp160687007-17883) [ ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:40419/solr/testExecute_shard1_replica_n2/ shard1
[junit4] 2> 960007 INFO (zkCallback-4773-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 960007 INFO (zkCallback-4772-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 960007 INFO (zkCallback-4772-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 960007 INFO (zkCallback-4773-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 960057 INFO (qtp160687007-17883) [ ] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 960058 INFO (qtp160687007-17883) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=testExecute_shard1_replica_n2&action=CREATE&numShards=1&collection=testExecute&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2030
[junit4] 2> 960107 INFO (qtp401615291-17886) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=testExecute_shard1_replica_n1&action=CREATE&numShards=1&collection=testExecute&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2067
[junit4] 2> 960108 INFO (qtp160687007-17879) [ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 960208 INFO (zkCallback-4773-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 960208 INFO (zkCallback-4772-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 960208 INFO (zkCallback-4772-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 960208 INFO (zkCallback-4773-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 961108 INFO (qtp160687007-17879) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=testExecute&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=3413
[junit4] 2> 961110 DEBUG (TEST-ExecutePlanActionTest.testExecute-seed#[B3D925FE12D9319C]) [ ] o.a.s.c.a.ExecutePlanAction -- processing event: {
[junit4] 2> "id":"14f0888173ea2680T8ne84nir2iyif81yhhktw44gb",
[junit4] 2> "source":"mock_trigger_name",
[junit4] 2> "eventTime":1508855964746000000,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[1508855964746000000],
[junit4] 2> "nodeNames":["127.0.0.1:40419_solr"]}} with context properties: {operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@65548c99, org.apache.solr.cloud.autoscaling.ExecutePlanActionTest$1@7c00b5ea]}
[junit4] 2> 961110 INFO (TEST-ExecutePlanActionTest.testExecute-seed#[B3D925FE12D9319C]) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testExecute&targetNode=127.0.0.1:33627_solr&replica=core_node4
[junit4] 2> 961111 DEBUG (TEST-ExecutePlanActionTest.testExecute-seed#[B3D925FE12D9319C]) [ ] o.a.s.c.a.ExecutePlanAction Saved requestId: mock_trigger_name/14f0888173ea2680T8ne84nir2iyif81yhhktw44gb/0 in znode: /autoscaling/triggerState/mock_trigger_name/execute_plan/op-0000000000
[junit4] 2> 961112 INFO (qtp160687007-17873) [ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :movereplica with params async=mock_trigger_name/14f0888173ea2680T8ne84nir2iyif81yhhktw44gb/0&replica=core_node4&action=MOVEREPLICA&collection=testExecute&targetNode=127.0.0.1:33627_solr&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 961112 INFO (qtp160687007-17873) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={async=mock_trigger_name/14f0888173ea2680T8ne84nir2iyif81yhhktw44gb/0&replica=core_node4&action=MOVEREPLICA&collection=testExecute&targetNode=127.0.0.1:33627_solr&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 961113 INFO (qtp160687007-17883) [ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=mock_trigger_name/14f0888173ea2680T8ne84nir2iyif81yhhktw44gb/0&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 961113 INFO (qtp160687007-17883) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=mock_trigger_name/14f0888173ea2680T8ne84nir2iyif81yhhktw44gb/0&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 961114 INFO (OverseerThreadFactory-4145-thread-2) [ ] o.a.s.c.MoveReplicaCmd Replica will be moved core_node4:{"core":"testExecute_shard1_replica_n2","base_url":"http://127.0.0.1:40419/solr","node_name":"127.0.0.1:40419_solr","state":"active","type":"NRT","leader":"true"}
[junit4] 2> 961114 INFO (OverseerCollectionConfigSetProcessor-98884384255901702-127.0.0.1:33627_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 961115 INFO (OverseerThreadFactory-4145-thread-2) [ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:33627_solr for creating new replica
[junit4] 2> 961115 INFO (OverseerStateUpdate-98884384255901702-127.0.0.1:33627_solr-n_0000000000) [ ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"addreplica",
[junit4] 2> "collection":"testExecute",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"testExecute_shard1_replica_n5",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:33627/solr",
[junit4] 2> "node_name":"127.0.0.1:33627_solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 961216 INFO (zkCallback-4773-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 961216 INFO (zkCallback-4772-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 961216 INFO (zkCallback-4772-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 961216 INFO (zkCallback-4773-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testExecute/state.json] for collection [testExecute] has occurred - updating... (live nodes size: [2])
[junit4] 2> 961316 INFO (qtp401615291-17888) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={async=mock_trigger_name/14f0888173ea2680T8ne84nir2iyif81yhhktw44gb/04774814269772243&qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf&name=testExecute_shard1_replica_n5&action=CREATE&collection=testExecute&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=0
[junit4] 2> 961316 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.h.a.CoreAdminOperation core create command async=mock_trigger_name/14f0888173ea2680T8ne84nir2iyif81yhhktw44gb/04774814269772243&qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf&name=testExecute_shard1_replica_n5&action=CREATE&collection=testExecute&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 961317 INFO (qtp401615291-17878) [ ] o.a.s.h.a.CoreAdminOperation Checking request status for : mock_trigger_name/14f0888173ea2680T8ne84nir2iyif81yhhktw44gb/04774814269772243
[junit4] 2> 961317 INFO (qtp401615291-17878) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&requestid=mock_trigger_name/14f0888173ea2680T8ne84nir2iyif81yhhktw44gb/04774814269772243&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 961320 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 961324 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.s.IndexSchema [testExecute_shard1_replica_n5] Schema name=minimal
[junit4] 2> 961325 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 961325 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.c.CoreContainer Creating SolrCore 'testExecute_shard1_replica_n5' using configuration from collection testExecute, trusted=true
[junit4] 2> 961326 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33627.solr.core.testExecute.shard1.replica_n5' (registry 'solr.core.testExecute.shard1.replica_n5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@fbf6221
[junit4] 2> 961326 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 961326 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.c.SolrCore [[testExecute_shard1_replica_n5] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ExecutePlanActionTest_B3D925FE12D9319C-001/tempDir-001/node2/testExecute_shard1_replica_n5], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ExecutePlanActionTest_B3D925FE12D9319C-001/tempDir-001/node2/./testExecute_shard1_replica_n5/data/]
[junit4] 2> 961351 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 961351 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 961352 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 961352 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 961353 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@2c8537f7[testExecute_shard1_replica_n5] main]
[junit4] 2> 961353 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 961353 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 961354 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 961354 INFO (searcherExecutor-4160-thread-1) [ ] o.a.s.c.SolrCore [testExecute_shard1_replica_n5] Registered new searcher Searcher@2c8537f7[testExecute_shard1_replica_n5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 961354 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582150152345354240
[junit4] 2> 961356 INFO (parallelCoreAdminExecutor-4146-thread-1) [ ] o.a.s.c.ZkController Core needs to recover:testExecute_shard1_replica_n5
[junit4] 2> 961356 INFO (updateExecutor-4767-thread-1) [ ] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 961356 INFO (recoveryExecutor-4769-thread-1) [ ] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 961356 INFO (recoveryExecutor-4769-thread-1) [ ] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 961357 INFO (recoveryExecutor-4769-thread-1) [ ] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[testExecute_shard1_replica_n5]
[junit4] 2> 961357 INFO (recoveryExecutor-4769-thread-1) [ ] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 961357 INFO (recoveryExecutor-4769-thread-1) [ ] o.a.s.c.RecoveryStrategy Publishing state of core [testExecute_shard1_replica_n5] as recovering, leader is [http://127.0.0.1:40419/solr/testExecute_shard1_replica_n2/] and I am [http://127.0.0.1:33627/solr/testExecute_shard1_replica_n5/]
[junit4] 2> 961358 INFO (recoveryExecutor-4769-thread-1) [ ] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:40419/solr]; [WaitForState: action=PREPRECOVERY&core=testExecute_shard1_replica_n2&nodeName=127.0.0.1:33627_solr&coreNodeName=core_node6&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 961358 INFO (qtp160687007-17874) [ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node6, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, maxTime: 183 s
[junit4] 2> 961359 INFO (qtp160687007-17874) [ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=testExecute, shard=shard1, thisCore=testExecute_shard1_replica_n2, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:33627_solr, coreNodeName=core_node6, onlyIfActiveCheckResult=false, nodeProps: core_node6:{"core":"testExecute_shard1_replica_n5","base_url":"http:/
[...truncated too long message...]
1152)
[junit4] > at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
[junit4] > at java.base@9/java.lang.Thread.run(Thread.java:844)
[junit4] > 2) Thread[id=22057, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[B3D925FE12D9319C]-SendThread(127.0.0.1:32793), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at java.base@9/java.lang.Thread.sleep(Native Method)
[junit4] > at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > 3) Thread[id=22193, name=zkCallback-5427-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
[junit4] > at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
[junit4] > at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
[junit4] > at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
[junit4] > at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
[junit4] > at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
[junit4] > at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
[junit4] > at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
[junit4] > at java.base@9/java.lang.Thread.run(Thread.java:844)
[junit4] > 4) Thread[id=22260, name=zkCallback-5427-thread-6, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
[junit4] > at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
[junit4] > at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
[junit4] > at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
[junit4] > at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
[junit4] > at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
[junit4] > at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
[junit4] > at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
[junit4] > at java.base@9/java.lang.Thread.run(Thread.java:844)
[junit4] > 5) Thread[id=22194, name=zkCallback-5427-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
[junit4] > at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
[junit4] > at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
[junit4] > at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
[junit4] > at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
[junit4] > at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
[junit4] > at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
[junit4] > at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
[junit4] > at java.base@9/java.lang.Thread.run(Thread.java:844)
[junit4] > 6) Thread[id=22192, name=zkCallback-5427-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
[junit4] > at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
[junit4] > at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
[junit4] > at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
[junit4] > at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
[junit4] > at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
[junit4] > at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
[junit4] > at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
[junit4] > at java.base@9/java.lang.Thread.run(Thread.java:844)
[junit4] > 7) Thread[id=22223, name=zkCallback-5427-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at java.base@9/jdk.internal.misc.Unsafe.park(Native Method)
[junit4] > at java.base@9/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
[junit4] > at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
[junit4] > at java.base@9/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
[junit4] > at java.base@9/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
[junit4] > at java.base@9/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1091)
[junit4] > at java.base@9/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
[junit4] > at java.base@9/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
[junit4] > at java.base@9/java.lang.Thread.run(Thread.java:844)
[junit4] > at __randomizedtesting.SeedInfo.seed([B3D925FE12D9319C]:0)
[junit4] Completed [502/744 (3!)] on J1 in 113.73s, 1 test, 3 errors <<< FAILURES!
[...truncated 3722 lines...]
[junit4] Suite: org.apache.solr.analytics.legacy.facet.LegacyFieldFacetCloudTest
[junit4] 2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/contrib/solr-analytics/test/J0/temp/solr.analytics.legacy.facet.LegacyFieldFacetCloudTest_144D2689CA067CE2-001/init-core-data-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=669, maxMBSortInHeap=6.097356916499175, sim=RandomSimilarity(queryNorm=false): {}, locale=lv-LV, timezone=Asia/Omsk
[junit4] 2> NOTE: Linux 4.10.0-33-generic amd64/Oracle Corporation 9 (64-bit)/cpus=8,threads=1,free=479123096,total=536870912
[junit4] 2> NOTE: All tests run in this JVM: [DateMathFunctionTest, LegacyFieldFacetTest, LegacyAbstractAnalyticsFacetCloudTest, CastingBooleanValueStreamTest, GTFunctionTest, RemoveFunctionTest, LTFunctionTest, BottomFunctionTest, LegacyFieldFacetCloudTest]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=LegacyFieldFacetCloudTest -Dtests.seed=144D2689CA067CE2 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=lv-LV -Dtests.timezone=Asia/Omsk -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J0 | LegacyFieldFacetCloudTest (suite) <<<
[junit4] > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44781/solr: Could not fully create collection: collection1
[junit4] > at __randomizedtesting.SeedInfo.seed([144D2689CA067CE2]:0)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
[junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
[junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
[junit4] > at org.apache.solr.analytics.legacy.LegacyAbstractAnalyticsCloudTest.setupCollection(LegacyAbstractAnalyticsCloudTest.java:50)
[junit4] > at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] > at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] > at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] > at java.base/java.lang.reflect.Method.invoke(Method.java:564)
[junit4] > at java.base/java.lang.Thread.run(Thread.java:844)
[junit4] Completed [48/78 (1!)] on J0 in 45.72s, 0 tests, 1 error <<< FAILURES!
[...truncated 37113 lines...]
[JENKINS] Lucene-Solr-7.x-Linux (32bit/jdk1.8.0_144) - Build # 658
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/658/
Java: 32bit/jdk1.8.0_144 -client -XX:+UseG1GC
8 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestSQLHandler
Error Message:
40 threads leaked from SUITE scope at org.apache.solr.handler.TestSQLHandler: 1) Thread[id=3659, name=zkCallback-565-thread-4, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=3640, name=Scheduler-11256806, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=3601, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=3621, name=qtp31360058-3621, state=RUNNABLE, group=TGRP-TestSQLHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=3638, name=Thread-1058, state=WAITING, group=TGRP-TestSQLHandler] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 6) Thread[id=3593, name=qtp14096919-3593-acceptor-0@94605d-ServerConnector@3f8e19{SSL,[ssl, http/1.1]}{127.0.0.1:41603}, state=RUNNABLE, group=TGRP-TestSQLHandler] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=3607, name=TEST-TestSQLHandler.doTest-seed#[9B2D40DD275224C5]-EventThread, state=WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 8) Thread[id=3630, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=3594, name=qtp14096919-3594, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=3612, name=Scheduler-27536433, state=WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=3626, name=org.eclipse.jetty.server.session.HashSessionManager@17d0edcTimer, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=3658, name=zkCallback-559-thread-3, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=3606, name=TEST-TestSQLHandler.doTest-seed#[9B2D40DD275224C5]-SendThread(127.0.0.1:36501), state=TIMED_WAITING, group=TGRP-TestSQLHandler] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 14) Thread[id=3609, name=Thread-1049, state=WAITING, group=TGRP-TestSQLHandler] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 15) Thread[id=3631, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=3617, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=3589, name=qtp14096919-3589, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=3654, name=zkCallback-559-thread-2-processing-n:127.0.0.1:41603_, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320) at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:93) at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:78) at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:113) at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:105) at org.apache.solr.cloud.OverseerTaskQueue.<init>(OverseerTaskQueue.java:52) at org.apache.solr.cloud.Overseer.getCollectionQueue(Overseer.java:715) at org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.<init>(OverseerCollectionConfigSetProcessor.java:48) at org.apache.solr.cloud.Overseer.start(Overseer.java:523) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:799) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56) at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348) at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269) at org.apache.solr.common.cloud.SolrZkClient$3$$Lambda$275/29378451.run(Unknown Source) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$115/16010626.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=3635, name=TEST-TestSQLHandler.doTest-seed#[9B2D40DD275224C5]-SendThread(127.0.0.1:36501), state=TIMED_WAITING, group=TGRP-TestSQLHandler] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 20) Thread[id=3624, name=qtp31360058-3624, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:812) at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:781) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1178) at org.apache.solr.cloud.ZkController.register(ZkController.java:1079) at org.apache.solr.cloud.ZkController.register(ZkController.java:1014) at org.apache.solr.core.ZkContainer.lambda$registerInZk$0(ZkContainer.java:181) at org.apache.solr.core.ZkContainer$$Lambda$194/18580022.run(Unknown Source) at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:208) at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:886) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1046) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:948) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91) at org.apache.solr.handler.admin.CoreAdminOperation$$Lambda$76/21289992.execute(Unknown Source) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) at org.eclipse.jetty.server.Server.handle(Server.java:534) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=3661, name=zkCallback-565-thread-5, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=3596, name=qtp14096919-3596, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=3625, name=qtp31360058-3625, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=3618, name=qtp31360058-3618, state=RUNNABLE, group=TGRP-TestSQLHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=3590, name=qtp14096919-3590, state=RUNNABLE, group=TGRP-TestSQLHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=3619, name=qtp31360058-3619, state=RUNNABLE, group=TGRP-TestSQLHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=3636, name=TEST-TestSQLHandler.doTest-seed#[9B2D40DD275224C5]-EventThread, state=WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 28) Thread[id=3592, name=qtp14096919-3592, state=RUNNABLE, group=TGRP-TestSQLHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=3595, name=qtp14096919-3595, state=RUNNABLE, group=TGRP-TestSQLHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=3657, name=zkCallback-565-thread-3, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 31) Thread[id=3615, name=searcherExecutor-1722-thread-1, state=WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=3597, name=org.eclipse.jetty.server.session.HashSessionManager@1651cc9Timer, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=3644, name=searcherExecutor-1732-thread-1, state=WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 34) Thread[id=3622, name=qtp31360058-3622-acceptor-0@8be0a8-ServerConnector@14a257{SSL,[ssl, http/1.1]}{127.0.0.1:34469}, state=RUNNABLE, group=TGRP-TestSQLHandler] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=3660, name=zkCallback-559-thread-4, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=3620, name=qtp31360058-3620, state=RUNNABLE, group=TGRP-TestSQLHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 37) Thread[id=3649, name=qtp31360058-3649, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 38) Thread[id=3602, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 39) Thread[id=3591, name=qtp14096919-3591, state=RUNNABLE, group=TGRP-TestSQLHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 40) Thread[id=3662, name=zkCallback-559-thread-5, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 40 threads leaked from SUITE scope at org.apache.solr.handler.TestSQLHandler:
1) Thread[id=3659, name=zkCallback-565-thread-4, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=3640, name=Scheduler-11256806, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=3601, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=3621, name=qtp31360058-3621, state=RUNNABLE, group=TGRP-TestSQLHandler]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=3638, name=Thread-1058, state=WAITING, group=TGRP-TestSQLHandler]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678)
6) Thread[id=3593, name=qtp14096919-3593-acceptor-0@94605d-ServerConnector@3f8e19{SSL,[ssl, http/1.1]}{127.0.0.1:41603}, state=RUNNABLE, group=TGRP-TestSQLHandler]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=3607, name=TEST-TestSQLHandler.doTest-seed#[9B2D40DD275224C5]-EventThread, state=WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
8) Thread[id=3630, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=3594, name=qtp14096919-3594, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=3612, name=Scheduler-27536433, state=WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=3626, name=org.eclipse.jetty.server.session.HashSessionManager@17d0edcTimer, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=3658, name=zkCallback-559-thread-3, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=3606, name=TEST-TestSQLHandler.doTest-seed#[9B2D40DD275224C5]-SendThread(127.0.0.1:36501), state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
14) Thread[id=3609, name=Thread-1049, state=WAITING, group=TGRP-TestSQLHandler]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678)
15) Thread[id=3631, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
16) Thread[id=3617, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
17) Thread[id=3589, name=qtp14096919-3589, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
18) Thread[id=3654, name=zkCallback-559-thread-2-processing-n:127.0.0.1:41603_, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:93)
at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:78)
at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:113)
at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:105)
at org.apache.solr.cloud.OverseerTaskQueue.<init>(OverseerTaskQueue.java:52)
at org.apache.solr.cloud.Overseer.getCollectionQueue(Overseer.java:715)
at org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.<init>(OverseerCollectionConfigSetProcessor.java:48)
at org.apache.solr.cloud.Overseer.start(Overseer.java:523)
at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:799)
at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
at org.apache.solr.common.cloud.SolrZkClient$3$$Lambda$275/29378451.run(Unknown Source)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$115/16010626.run(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
19) Thread[id=3635, name=TEST-TestSQLHandler.doTest-seed#[9B2D40DD275224C5]-SendThread(127.0.0.1:36501), state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
20) Thread[id=3624, name=qtp31360058-3624, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:812)
at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:781)
at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1178)
at org.apache.solr.cloud.ZkController.register(ZkController.java:1079)
at org.apache.solr.cloud.ZkController.register(ZkController.java:1014)
at org.apache.solr.core.ZkContainer.lambda$registerInZk$0(ZkContainer.java:181)
at org.apache.solr.core.ZkContainer$$Lambda$194/18580022.run(Unknown Source)
at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:208)
at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:886)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1046)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:948)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
at org.apache.solr.handler.admin.CoreAdminOperation$$Lambda$76/21289992.execute(Unknown Source)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:534)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
21) Thread[id=3661, name=zkCallback-565-thread-5, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
22) Thread[id=3596, name=qtp14096919-3596, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
23) Thread[id=3625, name=qtp31360058-3625, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
24) Thread[id=3618, name=qtp31360058-3618, state=RUNNABLE, group=TGRP-TestSQLHandler]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
25) Thread[id=3590, name=qtp14096919-3590, state=RUNNABLE, group=TGRP-TestSQLHandler]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
26) Thread[id=3619, name=qtp31360058-3619, state=RUNNABLE, group=TGRP-TestSQLHandler]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
27) Thread[id=3636, name=TEST-TestSQLHandler.doTest-seed#[9B2D40DD275224C5]-EventThread, state=WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
28) Thread[id=3592, name=qtp14096919-3592, state=RUNNABLE, group=TGRP-TestSQLHandler]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
29) Thread[id=3595, name=qtp14096919-3595, state=RUNNABLE, group=TGRP-TestSQLHandler]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
30) Thread[id=3657, name=zkCallback-565-thread-3, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
31) Thread[id=3615, name=searcherExecutor-1722-thread-1, state=WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
32) Thread[id=3597, name=org.eclipse.jetty.server.session.HashSessionManager@1651cc9Timer, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
33) Thread[id=3644, name=searcherExecutor-1732-thread-1, state=WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
34) Thread[id=3622, name=qtp31360058-3622-acceptor-0@8be0a8-ServerConnector@14a257{SSL,[ssl, http/1.1]}{127.0.0.1:34469}, state=RUNNABLE, group=TGRP-TestSQLHandler]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
35) Thread[id=3660, name=zkCallback-559-thread-4, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
36) Thread[id=3620, name=qtp31360058-3620, state=RUNNABLE, group=TGRP-TestSQLHandler]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
37) Thread[id=3649, name=qtp31360058-3649, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
38) Thread[id=3602, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
39) Thread[id=3591, name=qtp14096919-3591, state=RUNNABLE, group=TGRP-TestSQLHandler]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
40) Thread[id=3662, name=zkCallback-559-thread-5, state=TIMED_WAITING, group=TGRP-TestSQLHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([9B2D40DD275224C5]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestSQLHandler
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=3659, name=zkCallback-565-thread-4, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=3640, name=Scheduler-11256806, state=WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=3618, name=qtp31360058-3618, state=RUNNABLE, group=TGRP-TestSQLHandler] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=3590, name=qtp14096919-3590, state=RUNNABLE, group=TGRP-TestSQLHandler] at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317) at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207) at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213) at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=3621, name=qtp31360058-3621, state=RUNNABLE, group=TGRP-TestSQLHandler] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=3592, name=qtp14096919-3592, state=RUNNABLE, group=TGRP-TestSQLHandler] at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317) at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207) at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213) at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=3595, name=qtp14096919-3595, state=RUNNABLE, group=TGRP-TestSQLHandler] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=3657, name=zkCallback-565-thread-3, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=3638, name=Thread-1058, state=WAITING, group=TGRP-TestSQLHandler] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 10) Thread[id=3615, name=searcherExecutor-1722-thread-1, state=WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=3597, name=org.eclipse.jetty.server.session.HashSessionManager@1651cc9Timer, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=3644, name=searcherExecutor-1732-thread-1, state=WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=3612, name=Scheduler-27536433, state=WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=3626, name=org.eclipse.jetty.server.session.HashSessionManager@17d0edcTimer, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=3658, name=zkCallback-559-thread-3, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=3660, name=zkCallback-559-thread-4, state=TIMED_WAITING, group=TGRP-TestSQLHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.uti
[...truncated too long message...]
)
[junit4] 2> 280864 ERROR (qtp4347675-939) [n:127.0.0.1:36095_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ZkContainer :org.apache.solr.common.SolrException: Error getting leader from zk for shard shard1
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1207)
[junit4] 2> at org.apache.solr.cloud.ZkController.register(ZkController.java:1079)
[junit4] 2> at org.apache.solr.cloud.ZkController.register(ZkController.java:1014)
[junit4] 2> at org.apache.solr.core.ZkContainer.lambda$registerInZk$0(ZkContainer.java:181)
[junit4] 2> at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:208)
[junit4] 2> at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:886)
[junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1046)
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:948)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] 2> at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.solr.common.SolrException: No registered leader was found after waiting for 1220000ms , collection: collection1 slice: shard1 saw state=null with live_nodes=[]
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:816)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:781)
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1178)
[junit4] 2> ... 42 more
[junit4] 2>
[junit4] 2> 280865 INFO (qtp4347675-939) [n:127.0.0.1:36095_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=209751
[junit4] 2> 280865 ERROR (SUITE-JdbcTest-seed#[CEF6A1698E4F90BF]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 280866 INFO (SUITE-JdbcTest-seed#[CEF6A1698E4F90BF]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40459 40459
[junit4] 2> 286282 INFO (Thread-247) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40459 40459
[junit4] 2> 286283 WARN (Thread-247) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/aliases.json
[junit4] 2> 3 /solr/clusterprops.json
[junit4] 2> 2 /solr/security.json
[junit4] 2> 2 /solr/configs/conf
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/collections/collection1/state.json
[junit4] 2> 3 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/live_nodes
[junit4] 2> 3 /solr/collections
[junit4] 2>
[junit4] 2> 286283 INFO (SUITE-JdbcTest-seed#[CEF6A1698E4F90BF]-worker) [ ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
[junit4] 2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=14644, maxDocsPerChunk=685, blockSize=1), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=14644, blockSize=1)), sim=RandomSimilarity(queryNorm=false): {}, locale=hi-IN, timezone=Africa/Conakry
[junit4] 2> NOTE: Linux 4.10.0-33-generic i386/Oracle Corporation 1.8.0_144 (32-bit)/cpus=8,threads=1,free=143705384,total=281018368
[junit4] 2> NOTE: All tests run in this JVM: [TestLBHttpSolrClient, AppendEvaluatorTest, FieldAnalysisResponseTest, TermsResponseTest, UniformDistributionEvaluatorTest, JDBCStreamTest, LengthEvaluatorTest, ConcatOperationTest, FloorEvaluatorTest, OrEvaluatorTest, TestXMLEscaping, SolrExampleStreamingTest, SolrZkClientTest, TestRetryUtil, TestV2Request, ConcurrentUpdateSolrClientTest, CorrelationEvaluatorTest, StreamExpressionToExpessionTest, ModuloEvaluatorTest, EqualToEvaluatorTest, RoundEvaluatorTest, ArcTangentEvaluatorTest, CommonAdminParamsTest, JdbcTest]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=JdbcTest -Dtests.seed=CEF6A1698E4F90BF -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=hi-IN -Dtests.timezone=Africa/Conakry -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J0 | JdbcTest (suite) <<<
[junit4] > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:36095/solr: create the collection time out:180s
[junit4] > at __randomizedtesting.SeedInfo.seed([CEF6A1698E4F90BF]:0)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
[junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
[junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
[junit4] > at org.apache.solr.client.solrj.io.sql.JdbcTest.setupCluster(JdbcTest.java:77)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] Completed [145/145 (1!)] on J0 in 216.14s, 0 tests, 1 error <<< FAILURES!
[...truncated 42138 lines...]