You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/02/01 02:32:46 UTC
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 646 -
Still unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/646/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC
1 tests failed.
FAILED: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail
Error Message:
expected:<200> but was:<404>
Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
at __randomizedtesting.SeedInfo.seed([1B4C218B8F6E85C8:73F314A15FF49724]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:141)
at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:295)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 11185 lines...]
[junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_1B4C218B8F6E85C8-001/init-core-data-001
[junit4] 2> 899283 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[1B4C218B8F6E85C8]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
[junit4] 2> 899287 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[1B4C218B8F6E85C8]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_1B4C218B8F6E85C8-001/tempDir-001
[junit4] 2> 899287 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[1B4C218B8F6E85C8]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 899287 INFO (Thread-1039) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 899287 INFO (Thread-1039) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 899387 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[1B4C218B8F6E85C8]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:48641
[junit4] 2> 899400 INFO (jetty-launcher-814-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 899400 INFO (jetty-launcher-814-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 899402 INFO (jetty-launcher-814-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7ce880ca{/solr,null,AVAILABLE}
[junit4] 2> 899402 INFO (jetty-launcher-814-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3f3d2499{HTTP/1.1,[http/1.1]}{127.0.0.1:47419}
[junit4] 2> 899403 INFO (jetty-launcher-814-thread-2) [ ] o.e.j.s.Server Started @902678ms
[junit4] 2> 899403 INFO (jetty-launcher-814-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=47419}
[junit4] 2> 899403 INFO (jetty-launcher-814-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@74a6a136{/solr,null,AVAILABLE}
[junit4] 2> 899403 ERROR (jetty-launcher-814-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 899403 INFO (jetty-launcher-814-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4e27457{HTTP/1.1,[http/1.1]}{127.0.0.1:34049}
[junit4] 2> 899403 INFO (jetty-launcher-814-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.5.0
[junit4] 2> 899403 INFO (jetty-launcher-814-thread-1) [ ] o.e.j.s.Server Started @902679ms
[junit4] 2> 899403 INFO (jetty-launcher-814-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 899403 INFO (jetty-launcher-814-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34049}
[junit4] 2> 899403 INFO (jetty-launcher-814-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 899403 INFO (jetty-launcher-814-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-02-01T01:33:15.907Z
[junit4] 2> 899403 ERROR (jetty-launcher-814-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 899404 INFO (jetty-launcher-814-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.5.0
[junit4] 2> 899404 INFO (jetty-launcher-814-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 899404 INFO (jetty-launcher-814-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 899404 INFO (jetty-launcher-814-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-02-01T01:33:15.908Z
[junit4] 2> 899411 INFO (jetty-launcher-814-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 899411 INFO (jetty-launcher-814-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 899425 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x159f74dea920001, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 899425 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x159f74dea920002, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 899427 INFO (jetty-launcher-814-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 899427 INFO (jetty-launcher-814-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48641/solr
[junit4] 2> 899429 INFO (jetty-launcher-814-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 899430 INFO (jetty-launcher-814-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48641/solr
[junit4] 2> 899469 INFO (jetty-launcher-814-thread-2) [n:127.0.0.1:47419_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:47419_solr
[junit4] 2> 899475 INFO (jetty-launcher-814-thread-2) [n:127.0.0.1:47419_solr ] o.a.s.c.Overseer Overseer (id=97380780985155589-127.0.0.1:47419_solr-n_0000000000) starting
[junit4] 2> 899491 INFO (jetty-launcher-814-thread-2) [n:127.0.0.1:47419_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47419_solr
[junit4] 2> 899498 INFO (zkCallback-825-thread-1-processing-n:127.0.0.1:34049_solr) [n:127.0.0.1:34049_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 899498 INFO (zkCallback-824-thread-1-processing-n:127.0.0.1:47419_solr) [n:127.0.0.1:47419_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 899500 INFO (jetty-launcher-814-thread-1) [n:127.0.0.1:34049_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34049_solr
[junit4] 2> 899522 INFO (zkCallback-825-thread-1-processing-n:127.0.0.1:34049_solr) [n:127.0.0.1:34049_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 899526 INFO (zkCallback-824-thread-2-processing-n:127.0.0.1:47419_solr) [n:127.0.0.1:47419_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 899602 INFO (jetty-launcher-814-thread-1) [n:127.0.0.1:34049_solr ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 899621 INFO (jetty-launcher-814-thread-1) [n:127.0.0.1:34049_solr ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
[junit4] 2> 899683 INFO (jetty-launcher-814-thread-2) [n:127.0.0.1:47419_solr ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 899683 INFO (jetty-launcher-814-thread-2) [n:127.0.0.1:47419_solr ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
[junit4] 2> 899754 INFO (jetty-launcher-814-thread-1) [n:127.0.0.1:34049_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 899754 INFO (jetty-launcher-814-thread-2) [n:127.0.0.1:47419_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 899791 INFO (jetty-launcher-814-thread-2-EventThread) [n:127.0.0.1:47419_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 899796 INFO (jetty-launcher-814-thread-1-EventThread) [n:127.0.0.1:34049_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 899979 INFO (jetty-launcher-814-thread-1) [n:127.0.0.1:34049_solr ] o.a.s.h.c.HttpShardHandlerFactory Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@57941a87
[junit4] 2> 899980 INFO (jetty-launcher-814-thread-1) [n:127.0.0.1:34049_solr ] o.a.s.u.UpdateShardHandler Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@57941a87
[junit4] 2> 899981 INFO (jetty-launcher-814-thread-1) [n:127.0.0.1:34049_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_1B4C218B8F6E85C8-001/tempDir-001/node1/.
[junit4] 2> 899982 INFO (jetty-launcher-814-thread-2) [n:127.0.0.1:47419_solr ] o.a.s.h.c.HttpShardHandlerFactory Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@3a75d3fe
[junit4] 2> 899982 INFO (jetty-launcher-814-thread-2) [n:127.0.0.1:47419_solr ] o.a.s.u.UpdateShardHandler Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@3a75d3fe
[junit4] 2> 899984 INFO (jetty-launcher-814-thread-2) [n:127.0.0.1:47419_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_1B4C218B8F6E85C8-001/tempDir-001/node2/.
[junit4] 2> 900687 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancel
[junit4] 2> 900795 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 900797 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48641/solr ready
[junit4] 2> 900804 ERROR (qtp227354587-4653) [ ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_1]
[junit4] 2> 900817 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x159f74dea92000a, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 900822 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 900825 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48641/solr ready
[junit4] 2> 900843 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 900864 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 900883 ERROR (qtp2034613574-4657) [ ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_2]
[junit4] 2> 900894 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 900895 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48641/solr ready
[junit4] 2> 900902 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 900906 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 900906 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancel
[junit4] 2> 900941 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenSolrClient
[junit4] 2> 900952 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 900962 INFO (qtp227354587-4650) [n:127.0.0.1:47419_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfEHigFaG1p1BwMEFH5uzVbhrHlSFnkCQK6kKOQUeQ-RB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 900963 INFO (qtp227354587-4650) [n:127.0.0.1:47419_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfEHigFaG1p1BwMEFH5uzVbhrHlSFnkCQK6kKOQUeQ-RB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 900964 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenSolrClient
[junit4] 2> 901009 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenVerify
[junit4] 2> 901020 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 901021 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48641/solr ready
[junit4] 2> 901028 INFO (qtp227354587-4649) [n:127.0.0.1:47419_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfFLigFaG1p1SwQEFJpIDI7pbABlKwyWpmL_J_OFo5ZeB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 901028 INFO (qtp227354587-4649) [n:127.0.0.1:47419_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfFLigFaG1p1SwQEFJpIDI7pbABlKwyWpmL_J_OFo5ZeB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 901036 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 901038 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48641/solr ready
[junit4] 2> 901042 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 901048 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 901049 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48641/solr ready
[junit4] 2> 901051 INFO (qtp227354587-4651) [n:127.0.0.1:47419_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfFLigFaG1p1SwQEFJpIDI7pbABlKwyWpmL_J_OFo5ZeB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 901052 INFO (qtp227354587-4651) [n:127.0.0.1:47419_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfFLigFaG1p1SwQEFJpIDI7pbABlKwyWpmL_J_OFo5ZeB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 901056 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 901056 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenVerify
[junit4] 2> 901079 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
[junit4] 2> 901094 WARN (qtp227354587-4653) [ ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
[junit4] 2> 901095 WARN (qtp227354587-4653) [ ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
[junit4] 2> 901104 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 901106 WARN (qtp2034613574-4659) [ ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
[junit4] 2> 901107 WARN (qtp2034613574-4659) [ ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
[junit4] 2> 901107 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 901118 WARN (qtp227354587-4654) [ ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
[junit4] 2> 901119 WARN (qtp227354587-4654) [ ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
[junit4] 2> 901119 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 901121 WARN (qtp2034613574-4662) [ ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
[junit4] 2> 901121 WARN (qtp2034613574-4662) [ ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
[junit4] 2> 901122 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 901122 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenewFail
[junit4] 2> 901164 INFO (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testZNodePaths
[junit4] 2> 901175 INFO (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testZNodePaths
[junit4] 2> 901199 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
[junit4] 2> 901243 INFO (qtp227354587-4653) [n:127.0.0.1:47419_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfIJigFaG1p2CQgEFBgVCvyoi_dJtEfPIy27rxf6P4RtB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 901243 INFO (qtp227354587-4653) [n:127.0.0.1:47419_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfIJigFaG1p2CQgEFBgVCvyoi_dJtEfPIy27rxf6P4RtB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 901254 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 901258 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 901259 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48641/solr ready
[junit4] 2> 901262 INFO (qtp227354587-4655) [n:127.0.0.1:47419_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfIJigFaG1p2CQgEFBgVCvyoi_dJtEfPIy27rxf6P4RtB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 901262 INFO (qtp227354587-4655) [n:127.0.0.1:47419_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfIJigFaG1p2CQgEFBgVCvyoi_dJtEfPIy27rxf6P4RtB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 901265 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 901288 INFO (qtp227354587-4646) [n:127.0.0.1:47419_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfJLigFaG1p2SwkEFLAMq5-LqdofYy7aUG5YXLnBmwpqB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 901288 INFO (qtp227354587-4646) [n:127.0.0.1:47419_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfJLigFaG1p2SwkEFLAMq5-LqdofYy7aUG5YXLnBmwpqB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 901291 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 901296 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 901298 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48641/solr ready
[junit4] 2> 901301 INFO (qtp227354587-4646) [n:127.0.0.1:47419_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfJLigFaG1p2SwkEFLAMq5-LqdofYy7aUG5YXLnBmwpqB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 901301 INFO (qtp227354587-4646) [n:127.0.0.1:47419_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfJLigFaG1p2SwkEFLAMq5-LqdofYy7aUG5YXLnBmwpqB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 901307 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 901309 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48641/solr ready
[junit4] 2> 901314 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 901331 INFO (qtp227354587-4650) [n:127.0.0.1:47419_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfJ9igFaG1p2fQoEFMvUcAeC1b2lxhMRjP2eyVlEeHQOB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 901331 INFO (qtp227354587-4650) [n:127.0.0.1:47419_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfJ9igFaG1p2fQoEFMvUcAeC1b2lxhMRjP2eyVlEeHQOB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 901334 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 901339 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 901340 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48641/solr ready
[junit4] 2> 901342 INFO (qtp227354587-4650) [n:127.0.0.1:47419_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfJ9igFaG1p2fQoEFMvUcAeC1b2lxhMRjP2eyVlEeHQOB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 901342 INFO (qtp227354587-4650) [n:127.0.0.1:47419_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfJ9igFaG1p2fQoEFMvUcAeC1b2lxhMRjP2eyVlEeHQOB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 901348 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 901349 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48641/solr ready
[junit4] 2> 901351 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 901369 INFO (qtp227354587-4654) [n:127.0.0.1:47419_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfKjigFaG1p2owsEFEV0HTIiaY3J-g_1tREcfhYgX3alB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 901370 INFO (qtp227354587-4654) [n:127.0.0.1:47419_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfKjigFaG1p2owsEFEV0HTIiaY3J-g_1tREcfhYgX3alB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 901372 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 901376 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 901378 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48641/solr ready
[junit4] 2> 901379 INFO (qtp227354587-4649) [n:127.0.0.1:47419_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfKjigFaG1p2owsEFEV0HTIiaY3J-g_1tREcfhYgX3alB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 901379 INFO (qtp227354587-4649) [n:127.0.0.1:47419_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVn3TfKjigFaG1p2owsEFEV0HTIiaY3J-g_1tREcfhYgX3alB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 901385 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 901386 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48641/solr ready
[junit4] 2> 901389 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=to, 01-tammikuuta-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: to, 01-tammikuuta-1970 00:00:00 GMT
[junit4] 2> 901390 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
[junit4] 2> 901424 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationOtherOp
[junit4] 2> 901428 INFO (qtp227354587-4649) [n:127.0.0.1:47419_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params op=someSolrOperation&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 901428 INFO (qtp227354587-4649) [n:127.0.0.1:47419_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={op=someSolrOperation&action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 901429 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[1B4C218B8F6E85C8]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=test&e=1485948797931&s=W9JEpFdHKSYYwrNjs6o3II9duFA="; Path=/; Domain=127.0.0.1; Expires=ke, 01-helmikuuta-2017 11:33:17 GMT; HttpOnly". Invalid 'expires' attribute: ke, 01-helmikuuta-2017 11:33:17 GMT
[junit4] 2> 901429 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationOtherOp
[junit4] 2> 901457 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancelFail
[junit4] 2> 901477 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[1B4C218B8F6E85C8]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancelFail
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSolrCloudWithDelegationTokens -Dtests.method=testDelegationTokenCancelFail -Dtests.seed=1B4C218B8F6E85C8 -Dtests.slow=true -Dtests.locale=fi -Dtests.timezone=Asia/Manila -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 0.05s J0 | TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail <<<
[junit4] > Throwable #1: java.lang.AssertionError: expected:<200> but was:<404>
[junit4] > at __randomizedtesting.SeedInfo.seed([1B4C218B8F6E85C8:73F314A15FF49724]:0)
[junit4] > at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:141)
[junit4] > at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:295)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 901481 INFO (jetty-launcher-814-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@3f3d2499{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 901481 INFO (jetty-launcher-814-thread-2) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@4e27457{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 901482 INFO (jetty-launcher-814-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1710116385
[junit4] 2> 901482 INFO (jetty-launcher-814-thread-2) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1871742459
[junit4] 2> 901491 INFO (jetty-launcher-814-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 901491 INFO (jetty-launcher-814-thread-1) [ ] o.a.s.c.Overseer Overseer (id=97380780985155589-127.0.0.1:47419_solr-n_0000000000) closing
[junit4] 2> 901491 INFO (OverseerStateUpdate-97380780985155589-127.0.0.1:47419_solr-n_0000000000) [n:127.0.0.1:47419_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:47419_solr
[junit4] 2> 901492 INFO (zkCallback-824-thread-3-processing-n:127.0.0.1:47419_solr) [n:127.0.0.1:47419_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 901492 ERROR (Thread[Thread-1058,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:34049_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 901503 INFO (jetty-launcher-814-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 901504 INFO (jetty-launcher-814-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@74a6a136{/solr,null,UNAVAILABLE}
[junit4] 2> 901504 ERROR (Thread[Thread-1056,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:47419_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 901515 INFO (jetty-launcher-814-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7ce880ca{/solr,null,UNAVAILABLE}
[junit4] 2> 901521 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[1B4C218B8F6E85C8]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:48641 48641
[junit4] 2> 901572 INFO (Thread-1039) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:48641 48641
[junit4] 2> 901573 WARN (Thread-1039) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 15 /solr/aliases.json
[junit4] 2> 15 /solr/clusterprops.json
[junit4] 2> 2 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 15 /solr/clusterstate.json
[junit4] 2> 4 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_7
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_6
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_9
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_8
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_1
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_2
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_5
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_4
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_12
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_11
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_10
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 15 /solr/live_nodes
[junit4] 2> 15 /solr/collections
[junit4] 2> 3 /solr/overseer/queue
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
[junit4] 2>
[junit4] 2> 901574 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[1B4C218B8F6E85C8]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_1B4C218B8F6E85C8-001
[junit4] 2> Feb 01, 2017 1:33:18 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 5 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=1695, maxMBSortInHeap=5.302468064515876, sim=RandomSimilarity(queryNorm=false,coord=crazy): {}, locale=fi, timezone=Asia/Manila
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=120797472,total=305303552
[junit4] 2> NOTE: All tests run in this JVM: [AnalysisErrorHandlingTest, SolrCoreTest, TestSolrIndexConfig, TestFastOutputStream, TestSolrConfigHandlerConcurrent, TestScoreJoinQPScore, BadComponentTest, VersionInfoTest, TestFieldCacheSortRandom, DistributedFacetPivotSmallAdvancedTest, AtomicUpdatesTest, TestManagedResourceStorage, RecoveryZkTest, TestCloudSchemaless, CdcrReplicationDistributedZkTest, TestLFUCache, DistributedQueueTest, ConjunctionSolrSpellCheckerTest, TestCorePropertiesReload, OverriddenZkACLAndCredentialsProvidersTest, ShufflingReplicaListTransformerTest, WrapperMergePolicyFactoryTest, DirectSolrConnectionTest, TestSSLRandomization, SolrJmxReporterTest, TestSubQueryTransformer, TestNoOpRegenerator, TestPivotHelperCode, SolrIndexMetricsTest, DocValuesMissingTest, TestSerializedLuceneMatchVersion, ZkControllerTest, TestConfigSetsAPIZkFailure, BadCopyFieldTest, TestTestInjection, TestUseDocValuesAsStored2, TestClassicSimilarityFactory, HttpPartitionTest, TestFastLRUCache, DOMUtilTest, SolrMetricReporterTest, InfoHandlerTest, TestManagedSchema, RulesTest, TestDownShardTolerantSearch, DeleteLastCustomShardedReplicaTest, HdfsRestartWhileUpdatingTest, TestReplicationHandlerBackup, SortSpecParsingTest, RequiredFieldsTest, CoreSorterTest, TestSolrQueryParserResource, XmlUpdateRequestHandlerTest, RemoteQueryErrorTest, ShardRoutingTest, TestCSVResponseWriter, OverseerStatusTest, TestConfigSetsAPI, TestSolrDynamicMBean, StressHdfsTest, DeleteStatusTest, PreAnalyzedUpdateProcessorTest, TestNRTOpen, TestStressRecovery, HdfsBasicDistributedZk2Test, SynonymTokenizerTest, SecurityConfHandlerTest, StatelessScriptUpdateProcessorFactoryTest, MoreLikeThisHandlerTest, TestIBSimilarityFactory, TestCloudPseudoReturnFields, BlockJoinFacetRandomTest, TestSolrCloudWithDelegationTokens]
[junit4] Completed [136/681 (1!)] on J0 in 3.33s, 8 tests, 1 failure <<< FAILURES!
[...truncated 64233 lines...]
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 648 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/648/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
1 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130) at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102) at sun.reflect.GeneratedConstructorAccessor159.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:737) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:799) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1049) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:914) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:807) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:904) at org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:555) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
at sun.reflect.GeneratedConstructorAccessor159.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:737)
at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:799)
at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1049)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:914)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:807)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:904)
at org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:555)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([A2547973D35AAD87]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:269)
at sun.reflect.GeneratedMethodAccessor25.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 11582 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A2547973D35AAD87-001/init-core-data-001
[junit4] 2> 1551826 INFO (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 1551827 INFO (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A2547973D35AAD87-001/tempDir-001
[junit4] 2> 1551827 INFO (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1551828 INFO (Thread-2672) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1551828 INFO (Thread-2672) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1551928 INFO (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:55792
[junit4] 2> 1551939 INFO (jetty-launcher-1450-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1551939 INFO (jetty-launcher-1450-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1551943 INFO (jetty-launcher-1450-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@33e3e7c4{/solr,null,AVAILABLE}
[junit4] 2> 1551943 INFO (jetty-launcher-1450-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@210546bd{/solr,null,AVAILABLE}
[junit4] 2> 1551944 INFO (jetty-launcher-1450-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5600472{HTTP/1.1,[http/1.1]}{127.0.0.1:59725}
[junit4] 2> 1551944 INFO (jetty-launcher-1450-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7450ffd2{HTTP/1.1,[http/1.1]}{127.0.0.1:62554}
[junit4] 2> 1551944 INFO (jetty-launcher-1450-thread-2) [ ] o.e.j.s.Server Started @1555598ms
[junit4] 2> 1551944 INFO (jetty-launcher-1450-thread-1) [ ] o.e.j.s.Server Started @1555599ms
[junit4] 2> 1551944 INFO (jetty-launcher-1450-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59725}
[junit4] 2> 1551944 INFO (jetty-launcher-1450-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=62554}
[junit4] 2> 1551944 ERROR (jetty-launcher-1450-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1551944 ERROR (jetty-launcher-1450-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1551944 INFO (jetty-launcher-1450-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.5.0
[junit4] 2> 1551945 INFO (jetty-launcher-1450-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.5.0
[junit4] 2> 1551945 INFO (jetty-launcher-1450-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1551945 INFO (jetty-launcher-1450-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1551945 INFO (jetty-launcher-1450-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-02-01T20:13:37.433Z
[junit4] 2> 1551945 INFO (jetty-launcher-1450-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1551945 INFO (jetty-launcher-1450-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1551945 INFO (jetty-launcher-1450-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-02-01T20:13:37.433Z
[junit4] 2> 1551974 INFO (jetty-launcher-1450-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1551978 INFO (jetty-launcher-1450-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1551984 INFO (jetty-launcher-1450-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 1551984 INFO (jetty-launcher-1450-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 1551984 INFO (jetty-launcher-1450-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55792/solr
[junit4] 2> 1551985 INFO (jetty-launcher-1450-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55792/solr
[junit4] 2> 1551990 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x159fb4fa2a50004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 1552025 INFO (jetty-launcher-1450-thread-2) [n:127.0.0.1:59725_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59725_solr
[junit4] 2> 1552026 INFO (jetty-launcher-1450-thread-1) [n:127.0.0.1:62554_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62554_solr
[junit4] 2> 1552027 INFO (jetty-launcher-1450-thread-2) [n:127.0.0.1:59725_solr ] o.a.s.c.Overseer Overseer (id=97385186414886917-127.0.0.1:59725_solr-n_0000000000) starting
[junit4] 2> 1552031 INFO (zkCallback-1461-thread-1-processing-n:127.0.0.1:62554_solr) [n:127.0.0.1:62554_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1552056 INFO (zkCallback-1460-thread-1-processing-n:127.0.0.1:59725_solr) [n:127.0.0.1:59725_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1552081 INFO (jetty-launcher-1450-thread-2) [n:127.0.0.1:59725_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59725_solr
[junit4] 2> 1552083 INFO (zkCallback-1461-thread-1-processing-n:127.0.0.1:62554_solr) [n:127.0.0.1:62554_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1552083 INFO (zkCallback-1460-thread-1-processing-n:127.0.0.1:59725_solr) [n:127.0.0.1:59725_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1552135 INFO (jetty-launcher-1450-thread-2) [n:127.0.0.1:59725_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A2547973D35AAD87-001/tempDir-001/node2/.
[junit4] 2> 1552205 INFO (jetty-launcher-1450-thread-1) [n:127.0.0.1:62554_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A2547973D35AAD87-001/tempDir-001/node1/.
[junit4] 2> 1552581 INFO (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1552582 INFO (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55792/solr ready
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 1552768 WARN (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 1552778 WARN (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1552781 INFO (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 1552808 INFO (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_solaris.vm_45816_hdfs____.cin22a/webapp
[junit4] 2> 1553628 INFO (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@solaris-vm:45816
[junit4] 2> 1553953 WARN (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1553961 INFO (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 1553975 INFO (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_34212_datanode____.b7s9xj/webapp
[junit4] 2> 1554557 INFO (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34212
[junit4] 2> 1554668 WARN (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1554670 INFO (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 1554700 INFO (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_55035_datanode____eznopl/webapp
[junit4] 2> 1554727 INFO (IPC Server handler 4 on 63339) [ ] BlockStateChange BLOCK* processReport: from storage DS-78779717-4fd1-43e2-841f-56153e753508 node DatanodeRegistration(127.0.0.1:61716, datanodeUuid=90d6f25b-b28a-4544-b8e0-4e06a8e25420, infoPort=57350, infoSecurePort=0, ipcPort=45192, storageInfo=lv=-56;cid=testClusterID;nsid=1877585307;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 1554727 INFO (IPC Server handler 4 on 63339) [ ] BlockStateChange BLOCK* processReport: from storage DS-51968579-06d0-4637-b53b-b9d87f93a549 node DatanodeRegistration(127.0.0.1:61716, datanodeUuid=90d6f25b-b28a-4544-b8e0-4e06a8e25420, infoPort=57350, infoSecurePort=0, ipcPort=45192, storageInfo=lv=-56;cid=testClusterID;nsid=1877585307;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 1555363 INFO (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:55035
[junit4] 2> 1555690 INFO (IPC Server handler 6 on 63339) [ ] BlockStateChange BLOCK* processReport: from storage DS-f99adf46-e151-4c1e-92d6-c53b74a622ae node DatanodeRegistration(127.0.0.1:32962, datanodeUuid=bedcee36-d5f1-4eec-b6c9-aa73b7241e82, infoPort=45738, infoSecurePort=0, ipcPort=35677, storageInfo=lv=-56;cid=testClusterID;nsid=1877585307;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 1555690 INFO (IPC Server handler 6 on 63339) [ ] BlockStateChange BLOCK* processReport: from storage DS-96b3b5b2-53d1-484d-86a5-bdab56bc552a node DatanodeRegistration(127.0.0.1:32962, datanodeUuid=bedcee36-d5f1-4eec-b6c9-aa73b7241e82, infoPort=45738, infoSecurePort=0, ipcPort=35677, storageInfo=lv=-56;cid=testClusterID;nsid=1877585307;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 1555839 INFO (TEST-HdfsRecoveryZkTest.test-seed#[A2547973D35AAD87]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 1555843 INFO (qtp1654222091-9974) [n:127.0.0.1:59725_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1555847 INFO (OverseerThreadFactory-4745-thread-1-processing-n:127.0.0.1:59725_solr) [n:127.0.0.1:59725_solr ] o.a.s.c.CreateCollectionCmd Create collection recoverytest
[junit4] 2> 1555965 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 1555967 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 1556079 INFO (zkCallback-1460-thread-1-processing-n:127.0.0.1:59725_solr) [n:127.0.0.1:59725_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1556079 INFO (zkCallback-1461-thread-1-processing-n:127.0.0.1:62554_solr) [n:127.0.0.1:62554_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1556979 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
[junit4] 2> 1557001 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
[junit4] 2> 1557010 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema [recoverytest_shard1_replica2] Schema name=minimal
[junit4] 2> 1557010 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema [recoverytest_shard1_replica1] Schema name=minimal
[junit4] 2> 1557012 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1557013 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1557014 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica2' using configuration from collection recoverytest
[junit4] 2> 1557015 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica1' using configuration from collection recoverytest
[junit4] 2> 1557024 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:63339/data
[junit4] 2> 1557024 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 1557025 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [[recoverytest_shard1_replica2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A2547973D35AAD87-001/tempDir-001/node2/recoverytest_shard1_replica2], dataDir=[hdfs://solaris-vm:63339/data/recoverytest/core_node2/data/]
[junit4] 2> 1557025 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7effe5d8
[junit4] 2> 1557025 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:63339/data/recoverytest/core_node2/data/snapshot_metadata
[junit4] 2> 1557030 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:63339/data
[junit4] 2> 1557031 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 1557031 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [[recoverytest_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A2547973D35AAD87-001/tempDir-001/node1/recoverytest_shard1_replica1], dataDir=[hdfs://solaris-vm:63339/data/recoverytest/core_node1/data/]
[junit4] 2> 1557031 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7effe5d8
[junit4] 2> 1557031 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:63339/data/recoverytest/core_node1/data/snapshot_metadata
[junit4] 2> 1557041 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 1557041 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 1557041 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new global HDFS BlockCache
[junit4] 2> 1557054 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 1557055 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:63339/data/recoverytest/core_node1/data
[junit4] 2> 1557075 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 1557075 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 1557165 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:63339/data/recoverytest/core_node1/data/index
[junit4] 2> 1557174 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 1557174 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 1557168 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 1557180 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:63339/data/recoverytest/core_node2/data
[junit4] 2> 1557194 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 1557214 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:63339/data/recoverytest/core_node2/data/index
[junit4] 2> 1557223 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 1557223 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 1557227 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 1557327 INFO (IPC Server handler 8 on 63339) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:61716 is added to blk_1073741825_1001{UCState=COMMITTED, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-f99adf46-e151-4c1e-92d6-c53b74a622ae:NORMAL:127.0.0.1:32962|RBW], ReplicaUC[[DISK]DS-78779717-4fd1-43e2-841f-56153e753508:NORMAL:127.0.0.1:61716|RBW]]} size 71
[junit4] 2> 1557327 INFO (IPC Server handler 9 on 63339) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:32962 is added to blk_1073741825_1001 size 71
[junit4] 2> 1557343 INFO (IPC Server handler 7 on 63339) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:32962 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-51968579-06d0-4637-b53b-b9d87f93a549:NORMAL:127.0.0.1:61716|RBW], ReplicaUC[[DISK]DS-f99adf46-e151-4c1e-92d6-c53b74a622ae:NORMAL:127.0.0.1:32962|RBW]]} size 0
[junit4] 2> 1557343 INFO (IPC Server handler 7 on 63339) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:61716 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-51968579-06d0-4637-b53b-b9d87f93a549:NORMAL:127.0.0.1:61716|RBW], ReplicaUC[[DISK]DS-f99adf46-e151-4c1e-92d6-c53b74a622ae:NORMAL:127.0.0.1:32962|RBW]]} size 0
[junit4] 2> 1557390 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 1557390 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1557390 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 1557420 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1557420 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1557452 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@6ccda6b0[recoverytest_shard1_replica2] main]
[junit4] 2> 1557453 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 1557454 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 1557454 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1557456 INFO (searcherExecutor-4750-thread-1-processing-n:127.0.0.1:59725_solr x:recoverytest_shard1_replica2 s:shard1 c:recoverytest) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [recoverytest_shard1_replica2] Registered new searcher Searcher@6ccda6b0[recoverytest_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1557470 WARN (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://solaris-vm:40388/solr3 due to exception: java.net.ConnectException: Call From solaris-vm/127.0.0.1 to solaris-vm:40388 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 1557471 WARN (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://solaris-vm:40388/solr3 due to exception: java.net.ConnectException: Call From solaris-vm/127.0.0.1 to solaris-vm:40388 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 1557471 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1558162988554256384
[junit4] 2> 1557480 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 1557588 INFO (zkCallback-1461-thread-1-processing-n:127.0.0.1:62554_solr) [n:127.0.0.1:62554_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1557589 INFO (zkCallback-1460-thread-2-processing-n:127.0.0.1:59725_solr) [n:127.0.0.1:59725_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1557773 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 1557773 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1557773 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 1557786 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1557786 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1557811 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@9c74a63[recoverytest_shard1_replica1] main]
[junit4] 2> 1557813 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 1557813 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 1557813 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1557816 INFO (searcherExecutor-4751-thread-1-processing-n:127.0.0.1:62554_solr x:recoverytest_shard1_replica1 s:shard1 c:recoverytest) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [recoverytest_shard1_replica1] Registered new searcher Searcher@9c74a63[recoverytest_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1557832 WARN (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://solaris-vm:40388/solr3 due to exception: java.net.ConnectException: Call From solaris-vm/127.0.0.1 to solaris-vm:40388 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 1557835 WARN (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://solaris-vm:40388/solr3 due to exception: java.net.ConnectException: Call From solaris-vm/127.0.0.1 to solaris-vm:40388 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 1557835 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1558162988935938048
[junit4] 2> 1557985 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1557985 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1557985 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/
[junit4] 2> 1557986 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica2 url=http://127.0.0.1:59725/solr START replicas=[http://127.0.0.1:62554/solr/recoverytest_shard1_replica1/] nUpdates=100
[junit4] 2> 1557986 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica2 url=http://127.0.0.1:59725/solr DONE. We have no versions. sync failed.
[junit4] 2> 1557990 INFO (qtp1770919203-9970) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.S.Request [recoverytest_shard1_replica1] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1557990 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 1557990 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 1558012 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/ shard1
[junit4] 2> 1558117 INFO (zkCallback-1461-thread-1-processing-n:127.0.0.1:62554_solr) [n:127.0.0.1:62554_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1558117 INFO (zkCallback-1460-thread-1-processing-n:127.0.0.1:59725_solr) [n:127.0.0.1:59725_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1558165 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1558168 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=2206
[junit4] 2> 1558275 INFO (zkCallback-1461-thread-1-processing-n:127.0.0.1:62554_solr) [n:127.0.0.1:62554_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1558275 INFO (zkCallback-1460-thread-3-processing-n:127.0.0.1:59725_solr) [n:127.0.0.1:59725_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1558849 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=2883
[junit4] 2> 1558854 INFO (qtp1654222091-9974) [n:127.0.0.1:59725_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1558954 INFO (zkCallback-1461-thread-1-processing-n:127.0.0.1:62554_solr) [n:127.0.0.1:62554_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1558954 INFO (zkCallback-1460-thread-2-processing-n:127.0.0.1:59725_solr) [n:127.0.0.1:59725_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1559851 INFO (OverseerCollectionConfigSetProcessor-97385186414886917-127.0.0.1:59725_solr-n_0000000000) [n:127.0.0.1:59725_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1559855 INFO (qtp1654222091-9974) [n:127.0.0.1:59725_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=4013
[junit4] 2> 1559856 INFO (TEST-HdfsRecoveryZkTest.test-seed#[A2547973D35AAD87]) [ ] o.a.s.c.RecoveryZkTest Indexing 700 documents
[junit4] 2> 1559963 INFO (qtp1770919203-9986) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-0 (1558162991073984512)]} 0 53
[junit4] 2> 1559966 INFO (qtp1770919203-9985) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-0 (1558162991072935936)]} 0 56
[junit4] 2> 1559966 INFO (qtp1654222091-9975) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 (1558162991073984512)]} 0 92
[junit4] 2> 1559967 INFO (qtp1654222091-9971) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 (1558162991072935936)]} 0 94
[junit4] 2> 1559991 INFO (qtp1770919203-9987) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-0 (-1558162991190376448)]} 0 4
[junit4] 2> 1559992 INFO (qtp1770919203-9983) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-0 (-1558162991191425024)]} 0 3
[junit4] 2> 1559992 INFO (qtp1654222091-9978) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-0 (-1558162991191425024)]} 0 6
[junit4] 2> 1559995 INFO (qtp1654222091-9979) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-0 (-1558162991190376448)]} 0 11
[junit4] 2> 1560001 INFO (qtp1770919203-9970) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-1 (1558162991204007937)]} 0 1
[junit4] 2> 1560001 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-1 (1558162991204007936)]} 0 1
[junit4] 2> 1560001 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 (1558162991204007937)]} 0 3
[junit4] 2> 1560001 INFO (qtp1654222091-9974) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 (1558162991204007936)]} 0 3
[junit4] 2> 1560097 INFO (qtp1770919203-9986) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-2 (1558162991210299392)]} 0 20
[junit4] 2> 1560097 INFO (qtp1654222091-9975) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 (1558162991210299392)]} 0 93
[junit4] 2> 1560097 INFO (qtp1770919203-9981) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-2 (1558162991209250816)]} 0 29
[junit4] 2> 1560097 INFO (qtp1654222091-9977) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 (1558162991209250816)]} 0 94
[junit4] 2> 1560103 INFO (qtp1770919203-9987) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-3 (1558162991310962688)]} 0 0
[junit4] 2> 1560103 INFO (qtp1770919203-9985) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-3 (1558162991309914112)]} 0 1
[junit4] 2> 1560103 INFO (qtp1654222091-9978) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 (1558162991310962688)]} 0 3
[junit4] 2> 1560103 INFO (qtp1654222091-9971) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 (1558162991309914112)]} 0 3
[junit4] 2> 1560107 INFO (qtp1770919203-9983) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-4 (1558162991316205568)]} 0 0
[junit4] 2> 1560108 INFO (qtp1654222091-9979) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 (1558162991316205568)]} 0 2
[junit4] 2> 1560108 INFO (qtp1770919203-9983) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-4 (1558162991316205569)]} 0 0
[junit4] 2> 1560109 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 (1558162991316205569)]} 0 3
[junit4] 2> 1560113 INFO (qtp1770919203-9983) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-2 (-1558162991321448448)]} 0 1
[junit4] 2> 1560113 INFO (qtp1654222091-9974) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-2 (-1558162991321448448)]} 0 3
[junit4] 2> 1560113 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-2 (-1558162991321448449)]} 0 0
[junit4] 2> 1560113 INFO (qtp1654222091-9975) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-2 (-1558162991321448449)]} 0 2
[junit4] 2> 1560129 INFO (qtp1770919203-9986) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-5 (1558162991329837057)]} 0 1
[junit4] 2> 1560129 INFO (qtp1770919203-9981) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-5 (1558162991329837056)]} 0 1
[junit4] 2> 1560129 INFO (qtp1654222091-9977) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-5 (1558162991329837056)]} 0 11
[junit4] 2> 1560129 INFO (qtp1654222091-9978) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-5 (1558162991329837057)]} 0 11
[junit4] 2> 1560135 INFO (qtp1770919203-9987) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-6 (1558162991343468544)]} 0 1
[junit4] 2> 1560135 INFO (qtp1770919203-9985) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-6 (1558162991344517120)]} 0 1
[junit4] 2> 1560135 INFO (qtp1654222091-9979) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-6 (1558162991344517120)]} 0 3
[junit4] 2> 1560135 INFO (qtp1654222091-9971) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-6 (1558162991343468544)]} 0 4
[junit4] 2> 1560140 INFO (qtp1770919203-9970) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-3 (-1558162991349760000)]} 0 1
[junit4] 2> 1560140 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-3 (-1558162991349760000)]} 0 3
[junit4] 2> 1560142 INFO (qtp1770919203-9983) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-3 (-1558162991350808576)]} 0 1
[junit4] 2> 1560142 INFO (qtp1654222091-9974) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-3 (-1558162991350808576)]} 0 3
[junit4] 2> 1560149 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-7 (1558162991357100032)]} 0 3
[junit4] 2> 1560150 INFO (qtp1654222091-9975) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-7 (1558162991357100032)]} 0 6
[junit4] 2> 1560151 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-7 (1558162991357100033)]} 0 1
[junit4] 2> 1560151 INFO (qtp1654222091-9977) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-7 (1558162991357100033)]} 0 6
[junit4] 2> 1560177 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-5 (-1558162991365488640)]} 0 19
[junit4] 2> 1560178 INFO (qtp1654222091-9978) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-5 (-1558162991365488640)]} 0 26
[junit4] 2> 1560216 INFO (qtp1770919203-9981) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-5 (-1558162991390654464)]} 0 38
[junit4] 2> 1560217 INFO (qtp1654222091-9979) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-5 (-1558162991390654464)]} 0 40
[junit4] 2> 1560217 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-8 (1558162991394848768)]} 0 30
[junit4] 2> 1560217 INFO (qtp1654222091-9971) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-8 (1558162991394848768)]} 0 37
[junit4] 2> 1560224 INFO (qtp1770919203-9970) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-8 (1558162991436791808)]} 0 0
[junit4] 2> 1560224 INFO (qtp1770919203-9985) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-6 (-1558162991435743232)]} 0 3
[junit4] 2> 1560224 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-6 (-1558162991435743232)]} 0 5
[junit4] 2> 1560224 INFO (qtp1654222091-9974) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-8 (1558162991436791808)]} 0 4
[junit4] 2> 1560229 INFO (qtp1770919203-9983) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-9 (1558162991443083264)]} 0 0
[junit4] 2> 1560229 INFO (qtp1770919203-9986) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-6 (-1558162991443083265)]} 0 0
[junit4] 2> 1560229 INFO (qtp1654222091-9977) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-6 (-1558162991443083265)]} 0 2
[junit4] 2> 1560229 INFO (qtp1654222091-9975) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-9 (1558162991443083264)]} 0 3
[junit4] 2> 1560234 INFO (qtp1770919203-9987) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-10 (1558162991448326145)]} 0 1
[junit4] 2> 1560234 INFO (qtp1770919203-9981) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-9 (1558162991448326144)]} 0 1
[junit4] 2> 1560235 INFO (qtp1654222091-9979) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-10 (1558162991448326145)]} 0 3
[junit4] 2> 1560235 INFO (qtp1654222091-9978) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-9 (1558162991448326144)]} 0 3
[junit4] 2> 1560239 INFO (qtp1770919203-9970) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-11 (1558162991454617601)]} 0 0
[junit4] 2> 1560239 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-10 (1558162991454617600)]} 0 1
[junit4] 2> 1560240 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-11 (1558162991454617601)]} 0 2
[junit4] 2> 1560240 INFO (qtp1654222091-9971) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-10 (1558162991454617600)]} 0 3
[junit4] 2> 1560244 INFO (qtp1770919203-9985) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-12 (1558162991459860480)]} 0 1
[junit4] 2> 1560244 INFO (qtp1654222091-9974) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-12 (1558162991459860480)]} 0 2
[junit4] 2> 1560245 INFO (qtp1770919203-9983) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-11 (1558162991459860481)]} 0 1
[junit4] 2> 1560246 INFO (qtp1654222091-9977) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-11 (1558162991459860481)]} 0 3
[junit4] 2> 1560250 INFO (qtp1770919203-9986) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-12 (1558162991465103360)]} 0 0
[junit4] 2> 1560250 INFO (qtp1770919203-9987) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-11 (-1558162991465103361)]} 0 0
[junit4] 2> 1560250 INFO (qtp1654222091-9975) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-12 (1558162991465103360)]} 0 2
[junit4] 2> 1560250 INFO (qtp1654222091-9979) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-11 (-1558162991465103361)]} 0 2
[junit4] 2> 1560254 INFO (qtp1770919203-9981) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-11 (-1558162991470346240)]} 0 1
[junit4] 2> 1560255 INFO (qtp1654222091-9978) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-11 (-1558162991470346240)]} 0 2
[junit4] 2> 1560255 INFO (qtp1770919203-9970) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-13 (1558162991470346241)]} 0 1
[junit4] 2> 1560256 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-13 (1558162991470346241)]} 0 3
[junit4] 2> 1560260 INFO (qtp1770919203-9985) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-13 (1558162991476637696)]} 0 0
[junit4] 2> 1560261 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-13 (-1558162991476637697)]} 0 0
[junit4] 2> 1560261 INFO (qtp1654222091-9971) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-13 (1558162991476637696)]} 0 4
[junit4] 2> 1560261 INFO (qtp1654222091-9974) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-13 (-1558162991476637697)]} 0 2
[junit4] 2> 1560265 INFO (qtp1770919203-9986) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-14 (1558162991481880577)]} 0 0
[junit4] 2> 1560265 INFO (qtp1770919203-9983) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-13 (-1558162991481880576)]} 0 1
[junit4] 2> 1560266 INFO (qtp1654222091-9975) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-14 (1558162991481880577)]} 0 2
[junit4] 2> 1560266 INFO (qtp1654222091-9977) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-13 (-1558162991481880576)]} 0 3
[junit4] 2> 1560271 INFO (qtp1770919203-9987) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-14 (1558162991487123457)]} 0 1
[junit4] 2> 1560271 INFO (qtp1770919203-9981) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-14 (-1558162991487123456)]} 0 1
[junit4] 2> 1560271 INFO (qtp1654222091-9978) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-14 (1558162991487123457)]} 0 3
[junit4] 2> 1560271 INFO (qtp1654222091-9979) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-14 (-1558162991487123456)]} 0 3
[junit4] 2> 1560275 INFO (qtp1770919203-9985) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-15 (1558162991492366337)]} 0 0
[junit4] 2> 1560275 INFO (qtp1770919203-9970) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-14 (-1558162991492366336)]} 0 1
[junit4] 2> 1560276 INFO (qtp1654222091-9971) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-15 (1558162991492366337)]} 0 2
[junit4] 2> 1560276 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-14 (-1558162991492366336)]} 0 3
[junit4] 2> 1560281 INFO (qtp1770919203-9986) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-15 (1558162991497609216)]} 0 1
[junit4] 2> 1560281 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-15 (-1558162991497609217)]} 0 1
[junit4] 2> 1560281 INFO (qtp1654222091-9975) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-15 (-1558162991497609217)]} 0 3
[junit4] 2> 1560282 INFO (qtp1654222091-9974) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-15 (1558162991497609216)]} 0 3
[junit4] 2> 1560293 INFO (qtp1770919203-9987) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-15 (-1558162991503900673)]} 0 7
[junit4] 2> 1560293 INFO (qtp1770919203-9983) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-16 (1558162991503900672)]} 0 7
[junit4] 2> 1560293 INFO (qtp1654222091-9977) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-16 (1558162991503900672)]} 0 9
[junit4] 2> 1560293 INFO (qtp1654222091-9978) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-15 (-1558162991503900673)]} 0 9
[junit4] 2> 1560301 INFO (qtp1770919203-9985) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-16 (1558162991518580736)]} 0 1
[junit4] 2> 1560301 INFO (qtp1770919203-9981) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-16 (-1558162991518580737)]} 0 1
[junit4] 2> 1560301 INFO (qtp1654222091-9979) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-16 (1558162991518580736)]} 0 3
[junit4] 2> 1560301 INFO (qtp1654222091-9971) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-16 (-1558162991518580737)]} 0 3
[junit4] 2> 1560306 INFO (qtp1770919203-9970) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-16 (-1558162991523823616)]} 0 1
[junit4] 2> 1560306 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-17 (1558162991523823617)]} 0 1
[junit4] 2> 1560306 INFO (qtp1654222091-9975) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-17 (1558162991523823617)]} 0 3
[junit4] 2> 1560306 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-16 (-1558162991523823616)]} 0 3
[junit4] 2> 1560311 INFO (qtp1770919203-9986) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-17 (1558162991529066496)]} 0 0
[junit4] 2> 1560311 INFO (qtp1654222091-9974) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-17 (1558162991529066496)]} 0 2
[junit4] 2> 1560311 INFO (qtp1770919203-9983) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-17 (-1558162991530115072)]} 0 1
[junit4] 2> 1560312 INFO (qtp1654222091-9977) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-17 (-1558162991530115072)]} 0 3
[junit4] 2> 1560316 INFO (qtp1770919203-9987) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-17 (-1558162991534309376)]} 0 1
[junit4] 2> 1560316 INFO (qtp1654222091-9978) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-17 (-1558162991534309376)]} 0 3
[junit4] 2> 1560316 INFO (qtp1770919203-9985) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-18 (1558162991534309377)]} 0 1
[junit4] 2> 1560316 INFO (qtp1654222091-9979) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-18 (1558162991534309377)]} 0 2
[junit4] 2> 1560321 INFO (qtp1770919203-9981) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-19 (1558162991539552257)]} 0 1
[junit4] 2> 1560321 INFO (qtp1654222091-9975) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-19 (1558162991539552257)]} 0 2
[junit4] 2> 1560321 INFO (qtp1770919203-9970) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-18 (1558162991539552256)]} 0 1
[junit4] 2> 1560321 INFO (qtp1654222091-9971) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-18 (1558162991539552256)]} 0 3
[junit4] 2> 1560326 INFO (qtp1770919203-9986) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-19 (1558162991544795137)]} 0 0
[junit4] 2> 1560326 INFO (qtp1770919203-9973) [n:127.0.0.1:62554_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59725/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-20 (1558162991544795136)]} 0 1
[junit4] 2> 1560326 INFO (qtp1654222091-9974) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-19 (1558162991544795137)]} 0 2
[junit4] 2> 1560326 INFO (qtp1654222091-9976) [n:127.0.0.1:59725_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFacto
[...truncated too long message...]
top(MetricsSystemImpl.java:213)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:198)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest.teardownClass(HdfsRecoveryZkTest.java:51)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> 1578166 INFO (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:55792 55792
[junit4] 2> 1578166 INFO (SUITE-HdfsRecoveryZkTest-seed#[A2547973D35AAD87]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {rnd_b=PostingsFormat(name=Memory doPackFST= true), _version_=PostingsFormat(name=Direct), a_t=PostingsFormat(name=Direct), a_i=PostingsFormat(name=Memory doPackFST= true), id=PostingsFormat(name=Memory doPackFST= true)}, docValues:{}, maxPointsInLeafNode=101, maxMBSortInHeap=6.523613933953778, sim=RandomSimilarity(queryNorm=false,coord=yes): {}, locale=uk, timezone=America/Los_Angeles
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=2,free=107652232,total=440401920
[junit4] 2> NOTE: All tests run in this JVM: [TestCodecSupport, AddBlockUpdateTest, SolrInfoMBeanTest, SpatialHeatmapFacetsTest, TestFieldTypeResource, TestInitQParser, ForceLeaderTest, TestAuthorizationFramework, TestExpandComponent, TestWriterPerf, TestTolerantUpdateProcessorCloud, ConfigSetsAPITest, TestRealTimeGet, BadIndexSchemaTest, TestSchemaVersionResource, JvmMetricsTest, TestManagedSchemaAPI, CollectionReloadTest, TestReloadAndDeleteDocs, CleanupOldIndexTest, DistributedFacetPivotLongTailTest, TestConfigSets, TestSystemIdResolver, TestFuzzyAnalyzedSuggestions, ChaosMonkeySafeLeaderTest, TestSearchPerf, DirectoryFactoryTest, DistributedSuggestComponentTest, TestSimpleQParserPlugin, DistribCursorPagingTest, ParsingFieldUpdateProcessorsTest, BinaryUpdateRequestHandlerTest, ConnectionReuseTest, TestReload, FieldAnalysisRequestHandlerTest, TestSolrQueryParser, TestObjectReleaseTracker, DateRangeFieldTest, TestCloudDeleteByQuery, SolrIndexConfigTest, TestDocSet, TlogReplayBufferedWhileIndexingTest, TestMacroExpander, SpellPossibilityIteratorTest, LoggingHandlerTest, DistributedDebugComponentTest, ZkStateWriterTest, DistributedVersionInfoTest, DistributedTermsComponentTest, MissingSegmentRecoveryTest, TestSchemaResource, TestNamedUpdateProcessors, ShardSplitTest, TestDefaultSearchFieldResource, TestValueSourceCache, HdfsDirectoryFactoryTest, BlockJoinFacetSimpleTest, TestRestManager, TestCloudRecovery, RequestHandlersTest, TestSolrConfigHandler, TestCollectionAPI, CreateCollectionCleanupTest, DistributedFacetPivotSmallTest, TestUtils, BasicZkTest, CollectionsAPISolrJTest, HighlighterMaxOffsetTest, DistributedIntervalFacetingTest, OutOfBoxZkACLAndCredentialsProvidersTest, NoCacheHeaderTest, OverseerCollectionConfigSetProcessorTest, SpellCheckCollatorTest, ClassificationUpdateProcessorTest, TestQuerySenderListener, LeaderElectionIntegrationTest, TestManagedSynonymFilterFactory, TestMergePolicyConfig, LukeRequestHandlerTest, RegexBoostProcessorTest, FullSolrCloudDistribCmdsTest, DistribJoinFromCollectionTest, LeaderElectionContextKeyTest, DistanceFunctionTest, HdfsDirectoryTest, TestRandomRequestDistribution, TestOnReconnectListenerSupport, TestStressUserVersions, URLClassifyProcessorTest, TestSolrCoreProperties, NotRequiredUniqueKeyTest, CircularListTest, DistributedQueryComponentCustomSortTest, TestInitParams, UpdateParamsTest, TestExportWriter, QueryParsingTest, HdfsRecoveryZkTest]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=HdfsRecoveryZkTest -Dtests.seed=A2547973D35AAD87 -Dtests.slow=true -Dtests.locale=uk -Dtests.timezone=America/Los_Angeles -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 0.00s J1 | HdfsRecoveryZkTest (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
[junit4] > at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
[junit4] > at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
[junit4] > at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
[junit4] > at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:737)
[junit4] > at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:799)
[junit4] > at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1049)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:914)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:807)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:904)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:555)
[junit4] > at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([A2547973D35AAD87]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:269)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] Completed [273/681 (1!)] on J1 in 57.44s, 1 test, 1 failure <<< FAILURES!
[...truncated 63844 lines...]
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 647 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/647/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC
3 tests failed.
FAILED: org.apache.solr.handler.TestConfigReload.test
Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:52190 within 30000 ms
Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:52190 within 30000 ms
at __randomizedtesting.SeedInfo.seed([512CAA37E78AF8D6:D97895ED4976952E]:0)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:182)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:111)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:98)
at org.apache.solr.cloud.AbstractDistribZkTestBase.printLayout(AbstractDistribZkTestBase.java:322)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.distribTearDown(AbstractFullDistribZkTestBase.java:1500)
at org.apache.solr.handler.TestConfigReload.distribTearDown(TestConfigReload.java:69)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:969)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:52190 within 30000 ms
at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:233)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:174)
... 38 more
FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestConfigReload
Error Message:
72 threads leaked from SUITE scope at org.apache.solr.handler.TestConfigReload: 1) Thread[id=11131, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestConfigReload] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:258) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=11185, name=qtp1943874028-11185, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=11202, name=qtp1487495214-11202, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=11198, name=qtp1487495214-11198, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=11183, name=qtp1943874028-11183-acceptor-0@642b1bd5-ServerConnector@6564cb20{HTTP/1.1,[http/1.1]}{127.0.0.1:52190}, state=RUNNABLE, group=TGRP-TestConfigReload] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=11159, name=qtp779002420-11159, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=11184, name=qtp1943874028-11184, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=11197, name=qtp1487495214-11197-acceptor-0@ff07d46-ServerConnector@64c7dafd{HTTP/1.1,[http/1.1]}{127.0.0.1:33839}, state=RUNNABLE, group=TGRP-TestConfigReload] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=11203, name=Scheduler-1141139679, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=11119, name=qtp1516237200-11119-acceptor-0@3327b1b5-ServerConnector@49d83c93{HTTP/1.1,[http/1.1]}{127.0.0.1:57496}, state=RUNNABLE, group=TGRP-TestConfigReload] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 11) Thread[id=11140, name=OverseerHdfsCoreFailoverThread-97383292623192068-127.0.0.1:57496__b%2Fvy-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139) at java.lang.Thread.run(Thread.java:745) 12) Thread[id=11191, name=org.eclipse.jetty.server.session.HashSessionManager@536aad9dTimer, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 13) Thread[id=11188, name=qtp1943874028-11188, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 14) Thread[id=11200, name=qtp1487495214-11200, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 15) Thread[id=11173, name=TEST-TestConfigReload.test-seed#[512CAA37E78AF8D6]-EventThread, state=WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 16) Thread[id=11195, name=qtp1487495214-11195, state=RUNNABLE, group=TGRP-TestConfigReload] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 17) Thread[id=11130, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestConfigReload] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:258) at java.lang.Thread.run(Thread.java:745) 18) Thread[id=11162, name=Scheduler-356403476, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 19) Thread[id=11168, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestConfigReload] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:258) at java.lang.Thread.run(Thread.java:745) 20) Thread[id=11196, name=qtp1487495214-11196, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 21) Thread[id=11124, name=qtp1516237200-11124, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 22) Thread[id=11207, name=qtp917329835-11207, state=RUNNABLE, group=TGRP-TestConfigReload] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 23) Thread[id=11108, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestConfigReload] at java.lang.Object.wait(Native Method) at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146) 24) Thread[id=11189, name=qtp1943874028-11189, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 25) Thread[id=11194, name=Scheduler-1894010830, state=WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 26) Thread[id=11161, name=qtp779002420-11161, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 27) Thread[id=11213, name=qtp917329835-11213, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 28) Thread[id=11186, name=qtp1943874028-11186, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 29) Thread[id=11163, name=org.eclipse.jetty.server.session.HashSessionManager@5cf77b14Timer, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 30) Thread[id=11212, name=qtp917329835-11212, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 31) Thread[id=11158, name=qtp779002420-11158, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 32) Thread[id=11157, name=qtp779002420-11157, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 33) Thread[id=11151, name=TEST-TestConfigReload.test-seed#[512CAA37E78AF8D6]-SendThread(127.0.0.1:52190), state=RUNNABLE, group=TGRP-TestConfigReload] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) 34) Thread[id=11215, name=Scheduler-596221048, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 35) Thread[id=11139, name=OverseerCollectionConfigSetProcessor-97383292623192068-127.0.0.1:57496__b%2Fvy-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:343) at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:227) at java.lang.Thread.run(Thread.java:745) 36) Thread[id=11152, name=TEST-TestConfigReload.test-seed#[512CAA37E78AF8D6]-EventThread, state=WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 37) Thread[id=11122, name=qtp1516237200-11122, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 38) Thread[id=11118, name=qtp1516237200-11118, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 39) Thread[id=11154, name=qtp779002420-11154, state=RUNNABLE, group=TGRP-TestConfigReload] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 40) Thread[id=11106, name=Thread-4542, state=WAITING, group=TGRP-TestConfigReload] at java.lang.Object.wait(Native Method) at java.lang.Thread.join(Thread.java:1249) at java.lang.Thread.join(Thread.java:1323) at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302) at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309) at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490) 41) Thread[id=11142, name=Thread-4551, state=WAITING, group=TGRP-TestConfigReload] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1435) 42) Thread[id=11180, name=searcherExecutor-3711-thread-1, state=WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 43) Thread[id=11121, name=qtp1516237200-11121, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 44) Thread[id=11187, name=qtp1943874028-11187, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 45) Thread[id=11172, name=TEST-TestConfigReload.test-seed#[512CAA37E78AF8D6]-SendThread(127.0.0.1:52190), state=RUNNABLE, group=TGRP-TestConfigReload] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) 46) Thread[id=11201, name=qtp1487495214-11201, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 47) Thread[id=11199, name=qtp1487495214-11199, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 48) Thread[id=11138, name=OverseerStateUpdate-97383292623192068-127.0.0.1:57496__b%2Fvy-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:138) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:179) at java.lang.Thread.run(Thread.java:745) 49) Thread[id=11123, name=qtp1516237200-11123, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 50) Thread[id=11160, name=qtp779002420-11160, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 51) Thread[id=11156, name=qtp779002420-11156, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 52) Thread[id=11110, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120) 53) Thread[id=11136, name=TEST-TestConfigReload.test-seed#[512CAA37E78AF8D6]-EventThread, state=WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 54) Thread[id=11214, name=qtp917329835-11214, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 55) Thread[id=11149, name=searcherExecutor-3700-thread-1, state=WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 56) Thread[id=11125, name=Scheduler-1229212317, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 57) Thread[id=11126, name=org.eclipse.jetty.server.session.HashSessionManager@6da8be70Timer, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 58) Thread[id=11107, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestConfigReload] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178) at java.lang.Thread.run(Thread.java:745) 59) Thread[id=11155, name=qtp779002420-11155-acceptor-0@57685998-ServerConnector@3b4f7540{HTTP/1.1,[http/1.1]}{127.0.0.1:52925}, state=RUNNABLE, group=TGRP-TestConfigReload] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 60) Thread[id=11209, name=qtp917329835-11209, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 61) Thread[id=11182, name=qtp1943874028-11182, state=RUNNABLE, group=TGRP-TestConfigReload] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 62) Thread[id=11216, name=org.eclipse.jetty.server.session.HashSessionManager@179cc196Timer, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 63) Thread[id=11211, name=qtp917329835-11211, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 64) Thread[id=11120, name=qtp1516237200-11120, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 65) Thread[id=11208, name=qtp917329835-11208-acceptor-0@170bb9bf-ServerConnector@4bd25fb0{HTTP/1.1,[http/1.1]}{127.0.0.1:37078}, state=RUNNABLE, group=TGRP-TestConfigReload] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 66) Thread[id=11167, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestConfigReload] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:258) at java.lang.Thread.run(Thread.java:745) 67) Thread[id=11210, name=qtp917329835-11210, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 68) Thread[id=11135, name=TEST-TestConfigReload.test-seed#[512CAA37E78AF8D6]-SendThread(127.0.0.1:52190), state=RUNNABLE, group=TGRP-TestConfigReload] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) 69) Thread[id=11117, name=qtp1516237200-11117, state=RUNNABLE, group=TGRP-TestConfigReload] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 70) Thread[id=11176, name=Thread-4560, state=WAITING, group=TGRP-TestConfigReload] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1435) 71) Thread[id=11190, name=Scheduler-981445842, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 72) Thread[id=11204, name=org.eclipse.jetty.server.session.HashSessionManager@36757879Timer, state=TIMED_WAITING, group=TGRP-TestConfigReload] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 72 threads leaked from SUITE scope at org.apache.solr.handler.TestConfigReload:
1) Thread[id=11131, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:258)
at java.lang.Thread.run(Thread.java:745)
2) Thread[id=11185, name=qtp1943874028-11185, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
3) Thread[id=11202, name=qtp1487495214-11202, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
4) Thread[id=11198, name=qtp1487495214-11198, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
5) Thread[id=11183, name=qtp1943874028-11183-acceptor-0@642b1bd5-ServerConnector@6564cb20{HTTP/1.1,[http/1.1]}{127.0.0.1:52190}, state=RUNNABLE, group=TGRP-TestConfigReload]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
6) Thread[id=11159, name=qtp779002420-11159, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
7) Thread[id=11184, name=qtp1943874028-11184, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
8) Thread[id=11197, name=qtp1487495214-11197-acceptor-0@ff07d46-ServerConnector@64c7dafd{HTTP/1.1,[http/1.1]}{127.0.0.1:33839}, state=RUNNABLE, group=TGRP-TestConfigReload]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
9) Thread[id=11203, name=Scheduler-1141139679, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
10) Thread[id=11119, name=qtp1516237200-11119-acceptor-0@3327b1b5-ServerConnector@49d83c93{HTTP/1.1,[http/1.1]}{127.0.0.1:57496}, state=RUNNABLE, group=TGRP-TestConfigReload]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
11) Thread[id=11140, name=OverseerHdfsCoreFailoverThread-97383292623192068-127.0.0.1:57496__b%2Fvy-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
at java.lang.Thread.run(Thread.java:745)
12) Thread[id=11191, name=org.eclipse.jetty.server.session.HashSessionManager@536aad9dTimer, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
13) Thread[id=11188, name=qtp1943874028-11188, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
14) Thread[id=11200, name=qtp1487495214-11200, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
15) Thread[id=11173, name=TEST-TestConfigReload.test-seed#[512CAA37E78AF8D6]-EventThread, state=WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
16) Thread[id=11195, name=qtp1487495214-11195, state=RUNNABLE, group=TGRP-TestConfigReload]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
17) Thread[id=11130, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:258)
at java.lang.Thread.run(Thread.java:745)
18) Thread[id=11162, name=Scheduler-356403476, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
19) Thread[id=11168, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:258)
at java.lang.Thread.run(Thread.java:745)
20) Thread[id=11196, name=qtp1487495214-11196, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
21) Thread[id=11124, name=qtp1516237200-11124, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
22) Thread[id=11207, name=qtp917329835-11207, state=RUNNABLE, group=TGRP-TestConfigReload]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
23) Thread[id=11108, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at java.lang.Object.wait(Native Method)
at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
24) Thread[id=11189, name=qtp1943874028-11189, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
25) Thread[id=11194, name=Scheduler-1894010830, state=WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
26) Thread[id=11161, name=qtp779002420-11161, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
27) Thread[id=11213, name=qtp917329835-11213, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
28) Thread[id=11186, name=qtp1943874028-11186, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
29) Thread[id=11163, name=org.eclipse.jetty.server.session.HashSessionManager@5cf77b14Timer, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
30) Thread[id=11212, name=qtp917329835-11212, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
31) Thread[id=11158, name=qtp779002420-11158, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
32) Thread[id=11157, name=qtp779002420-11157, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
33) Thread[id=11151, name=TEST-TestConfigReload.test-seed#[512CAA37E78AF8D6]-SendThread(127.0.0.1:52190), state=RUNNABLE, group=TGRP-TestConfigReload]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
34) Thread[id=11215, name=Scheduler-596221048, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
35) Thread[id=11139, name=OverseerCollectionConfigSetProcessor-97383292623192068-127.0.0.1:57496__b%2Fvy-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:343)
at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)
at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:227)
at java.lang.Thread.run(Thread.java:745)
36) Thread[id=11152, name=TEST-TestConfigReload.test-seed#[512CAA37E78AF8D6]-EventThread, state=WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
37) Thread[id=11122, name=qtp1516237200-11122, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
38) Thread[id=11118, name=qtp1516237200-11118, state=TIMED_WAITING, group=TGRP-TestConfigReload]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
39) Thread[id=11154, name=qtp779002420-11154, state=RUNNABLE, group=TGRP-TestConfigReload]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector
[...truncated too long message...]
readPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 15) Thread[id=11135, name=TEST-TestConfigReload.test-seed#[512CAA37E78AF8D6]-SendThread(127.0.0.1:52190), state=TIMED_WAITING, group=TGRP-TestConfigReload]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
[junit4] > 16) Thread[id=11162, name=Scheduler-356403476, state=TIMED_WAITING, group=TGRP-TestConfigReload]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 17) Thread[id=11154, name=qtp779002420-11154, state=RUNNABLE, group=TGRP-TestConfigReload]
[junit4] > at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
[junit4] > at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
[junit4] > at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 18) Thread[id=11207, name=qtp917329835-11207, state=RUNNABLE, group=TGRP-TestConfigReload]
[junit4] > at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
[junit4] > at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
[junit4] > at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 19) Thread[id=11142, name=Thread-4551, state=WAITING, group=TGRP-TestConfigReload]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1435)
[junit4] > 20) Thread[id=11117, name=qtp1516237200-11117, state=RUNNABLE, group=TGRP-TestConfigReload]
[junit4] > at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
[junit4] > at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
[junit4] > at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 21) Thread[id=11176, name=Thread-4560, state=WAITING, group=TGRP-TestConfigReload]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1435)
[junit4] > 22) Thread[id=11190, name=Scheduler-981445842, state=TIMED_WAITING, group=TGRP-TestConfigReload]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 23) Thread[id=11204, name=org.eclipse.jetty.server.session.HashSessionManager@36757879Timer, state=TIMED_WAITING, group=TGRP-TestConfigReload]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([512CAA37E78AF8D6]:0)
[junit4] Completed [235/681 (1!)] on J0 in 181.70s, 1 test, 3 errors <<< FAILURES!
[...truncated 63899 lines...]