You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2016/12/10 08:30:25 UTC
[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build #
1000 - Still Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1000/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC
1 tests failed.
FAILED: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail
Error Message:
expected:<200> but was:<404>
Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
at __randomizedtesting.SeedInfo.seed([5D4C2B793AE9A9E3:35F31E53EA73BB0F]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:140)
at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:294)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 11469 lines...]
[junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_5D4C2B793AE9A9E3-001/init-core-data-001
[junit4] 2> 1438044 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[5D4C2B793AE9A9E3]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 1438049 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[5D4C2B793AE9A9E3]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_5D4C2B793AE9A9E3-001/tempDir-001
[junit4] 2> 1438049 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[5D4C2B793AE9A9E3]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1438050 INFO (Thread-2870) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1438050 INFO (Thread-2870) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1438150 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[5D4C2B793AE9A9E3]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:43406
[junit4] 2> 1438158 INFO (jetty-launcher-1936-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1438158 INFO (jetty-launcher-1936-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1438161 INFO (jetty-launcher-1936-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7159a6e1{/solr,null,AVAILABLE}
[junit4] 2> 1438161 INFO (jetty-launcher-1936-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6a63c7fc{/solr,null,AVAILABLE}
[junit4] 2> 1438162 INFO (jetty-launcher-1936-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1dac68c6{HTTP/1.1,[http/1.1]}{127.0.0.1:40185}
[junit4] 2> 1438162 INFO (jetty-launcher-1936-thread-1) [ ] o.e.j.s.Server Started @1441536ms
[junit4] 2> 1438162 INFO (jetty-launcher-1936-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40185}
[junit4] 2> 1438162 ERROR (jetty-launcher-1936-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1438162 INFO (jetty-launcher-1936-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1438162 INFO (jetty-launcher-1936-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1438162 INFO (jetty-launcher-1936-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1438162 INFO (jetty-launcher-1936-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-10T07:45:06.904Z
[junit4] 2> 1438163 INFO (jetty-launcher-1936-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4ac53945{HTTP/1.1,[http/1.1]}{127.0.0.1:34833}
[junit4] 2> 1438163 INFO (jetty-launcher-1936-thread-2) [ ] o.e.j.s.Server Started @1441537ms
[junit4] 2> 1438163 INFO (jetty-launcher-1936-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34833}
[junit4] 2> 1438163 ERROR (jetty-launcher-1936-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1438163 INFO (jetty-launcher-1936-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1438163 INFO (jetty-launcher-1936-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1438163 INFO (jetty-launcher-1936-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1438163 INFO (jetty-launcher-1936-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-10T07:45:06.905Z
[junit4] 2> 1438168 INFO (jetty-launcher-1936-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1438168 INFO (jetty-launcher-1936-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1438177 INFO (jetty-launcher-1936-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43406/solr
[junit4] 2> 1438177 INFO (jetty-launcher-1936-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43406/solr
[junit4] 2> 1438224 INFO (jetty-launcher-1936-thread-1) [n:127.0.0.1:40185_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:40185_solr
[junit4] 2> 1438226 INFO (jetty-launcher-1936-thread-1) [n:127.0.0.1:40185_solr ] o.a.s.c.Overseer Overseer (id=97082140707586053-127.0.0.1:40185_solr-n_0000000000) starting
[junit4] 2> 1438233 INFO (jetty-launcher-1936-thread-2) [n:127.0.0.1:34833_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34833_solr
[junit4] 2> 1438235 INFO (zkCallback-1947-thread-1-processing-n:127.0.0.1:34833_solr) [n:127.0.0.1:34833_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1438244 INFO (OverseerStateUpdate-97082140707586053-127.0.0.1:40185_solr-n_0000000000) [n:127.0.0.1:40185_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1438245 INFO (jetty-launcher-1936-thread-1) [n:127.0.0.1:40185_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40185_solr
[junit4] 2> 1438248 INFO (zkCallback-1946-thread-2-processing-n:127.0.0.1:40185_solr) [n:127.0.0.1:40185_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1438250 INFO (zkCallback-1947-thread-1-processing-n:127.0.0.1:34833_solr) [n:127.0.0.1:34833_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1438281 INFO (jetty-launcher-1936-thread-1) [n:127.0.0.1:40185_solr ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 1438293 INFO (jetty-launcher-1936-thread-1) [n:127.0.0.1:40185_solr ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
[junit4] 2> 1438301 INFO (jetty-launcher-1936-thread-2) [n:127.0.0.1:34833_solr ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 1438302 INFO (jetty-launcher-1936-thread-2) [n:127.0.0.1:34833_solr ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
[junit4] 2> 1438387 INFO (jetty-launcher-1936-thread-1) [n:127.0.0.1:40185_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 1438387 INFO (jetty-launcher-1936-thread-2) [n:127.0.0.1:34833_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 1438413 INFO (jetty-launcher-1936-thread-1-EventThread) [n:127.0.0.1:40185_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 1438413 INFO (jetty-launcher-1936-thread-2-EventThread) [n:127.0.0.1:34833_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 1438551 INFO (jetty-launcher-1936-thread-1) [n:127.0.0.1:40185_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_5D4C2B793AE9A9E3-001/tempDir-001/node1/.
[junit4] 2> 1438557 INFO (jetty-launcher-1936-thread-2) [n:127.0.0.1:34833_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_5D4C2B793AE9A9E3-001/tempDir-001/node2/.
[junit4] 2> 1438599 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
[junit4] 2> 1438684 INFO (qtp2080320930-12171) [n:127.0.0.1:40185_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVEjigFZC73VIwEDFCGVbSpieT2oaulanJZhx3TU7vP2B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1438684 INFO (qtp2080320930-12171) [n:127.0.0.1:40185_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVEjigFZC73VIwEDFCGVbSpieT2oaulanJZhx3TU7vP2B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1438705 INFO (qtp1047257633-12166) [n:127.0.0.1:34833_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVEjigFZC73VIwEDFCGVbSpieT2oaulanJZhx3TU7vP2B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1438705 INFO (qtp1047257633-12166) [n:127.0.0.1:34833_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVEjigFZC73VIwEDFCGVbSpieT2oaulanJZhx3TU7vP2B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1438711 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1438713 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43406/solr ready
[junit4] 2> 1438744 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1438745 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43406/solr ready
[junit4] 2> 1438747 INFO (qtp1047257633-12169) [n:127.0.0.1:34833_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVGEigFZC73VhAIDFFOTtjonWvpywlQOSFZjff59_p1rB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1438748 INFO (qtp1047257633-12169) [n:127.0.0.1:34833_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVGEigFZC73VhAIDFFOTtjonWvpywlQOSFZjff59_p1rB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1438764 INFO (qtp1047257633-12176) [n:127.0.0.1:34833_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVGEigFZC73VhAIDFFOTtjonWvpywlQOSFZjff59_p1rB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1438764 INFO (qtp1047257633-12176) [n:127.0.0.1:34833_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVGEigFZC73VhAIDFFOTtjonWvpywlQOSFZjff59_p1rB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1438770 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1438771 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43406/solr ready
[junit4] 2> 1438786 INFO (qtp2080320930-12172) [n:127.0.0.1:40185_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVG9igFZC73VvQMDFDgbU7ByB4bpzBiXUjn3YGQhyuKAB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1438787 INFO (qtp2080320930-12172) [n:127.0.0.1:40185_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVG9igFZC73VvQMDFDgbU7ByB4bpzBiXUjn3YGQhyuKAB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1438792 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1438794 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43406/solr ready
[junit4] 2> 1438800 INFO (qtp1047257633-12176) [n:127.0.0.1:34833_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVG9igFZC73VvQMDFDgbU7ByB4bpzBiXUjn3YGQhyuKAB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1438800 INFO (qtp1047257633-12176) [n:127.0.0.1:34833_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVG9igFZC73VvQMDFDgbU7ByB4bpzBiXUjn3YGQhyuKAB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1438806 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1438807 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43406/solr ready
[junit4] 2> 1438823 INFO (qtp2080320930-12170) [n:127.0.0.1:40185_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVHhigFZC73V4QQDFJbnzQxavGPuQ-ch0HjDD-EPvInSB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1438823 INFO (qtp2080320930-12170) [n:127.0.0.1:40185_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVHhigFZC73V4QQDFJbnzQxavGPuQ-ch0HjDD-EPvInSB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1438828 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1438830 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43406/solr ready
[junit4] 2> 1438835 INFO (qtp1047257633-12163) [n:127.0.0.1:34833_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVHhigFZC73V4QQDFJbnzQxavGPuQ-ch0HjDD-EPvInSB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1438835 INFO (qtp1047257633-12163) [n:127.0.0.1:34833_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVHhigFZC73V4QQDFJbnzQxavGPuQ-ch0HjDD-EPvInSB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1438840 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1438842 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43406/solr ready
[junit4] 2> 1438845 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
[junit4] 2> 1438870 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenSolrClient
[junit4] 2> 1438879 INFO (qtp2080320930-12172) [n:127.0.0.1:40185_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVIcigFZC73WHAUDFJHrxtNcg-duV64R-hejG2EK2myxB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1438879 INFO (qtp2080320930-12172) [n:127.0.0.1:40185_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVIcigFZC73WHAUDFJHrxtNcg-duV64R-hejG2EK2myxB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1438880 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenSolrClient
[junit4] 2> 1438907 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
[junit4] 2> 1438913 WARN (qtp2080320930-12174) [ ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
[junit4] 2> 1438913 WARN (qtp2080320930-12174) [ ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
[junit4] 2> 1438914 WARN (qtp1047257633-12169) [n:127.0.0.1:34833_solr ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
[junit4] 2> 1438918 WARN (qtp1047257633-12169) [n:127.0.0.1:34833_solr ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
[junit4] 2> 1438923 WARN (qtp2080320930-12168) [ ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
[junit4] 2> 1438924 WARN (qtp2080320930-12168) [ ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
[junit4] 2> 1438925 WARN (qtp1047257633-12176) [n:127.0.0.1:34833_solr ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
[junit4] 2> 1438925 WARN (qtp1047257633-12176) [n:127.0.0.1:34833_solr ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
[junit4] 2> 1438925 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenewFail
[junit4] 2> 1438973 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenVerify
[junit4] 2> 1438989 INFO (qtp2080320930-12164) [n:127.0.0.1:40185_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVKEigFZC73WhAgDFMHNhNZGqDRqNRHGzOIeJVkKiDeYB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1438989 INFO (qtp2080320930-12164) [n:127.0.0.1:40185_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVKEigFZC73WhAgDFMHNhNZGqDRqNRHGzOIeJVkKiDeYB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1438994 INFO (qtp1047257633-12173) [n:127.0.0.1:34833_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVKEigFZC73WhAgDFMHNhNZGqDRqNRHGzOIeJVkKiDeYB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1438994 INFO (qtp1047257633-12173) [n:127.0.0.1:34833_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjnsVKEigFZC73WhAgDFMHNhNZGqDRqNRHGzOIeJVkKiDeYB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1439000 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1439002 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43406/solr ready
[junit4] 2> 1439004 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x158e7b14ee90011, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 1439005 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenVerify
[junit4] 2> 1439038 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancelFail
[junit4] 2> 1439063 ERROR (qtp1047257633-12163) [n:127.0.0.1:34833_solr ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_9]
[junit4] 2> 1439063 ERROR (qtp1047257633-12163) [n:127.0.0.1:34833_solr ] o.a.h.s.t.d.ZKDelegationTokenSecretManager Exception while checking if token exist !!
[junit4] 2> org.apache.hadoop.security.token.SecretManager$InvalidToken: Token not found
[junit4] 2> at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.cancelToken(AbstractDelegationTokenSecretManager.java:537)
[junit4] 2> at org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager.cancelToken(ZKDelegationTokenSecretManager.java:800)
[junit4] 2> at org.apache.hadoop.security.token.delegation.web.DelegationTokenManager.cancelToken(DelegationTokenManager.java:187)
[junit4] 2> at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.managementOperation(DelegationTokenAuthenticationHandler.java:264)
[junit4] 2> at org.apache.solr.security.KerberosPlugin$RequestContinuesRecorderAuthenticationHandler.managementOperation(KerberosPlugin.java:566)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:514)
[junit4] 2> at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:128)
[junit4] 2> at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
[junit4] 2> at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:265)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:362)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:268)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:254)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:113)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 1439064 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancelFail
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSolrCloudWithDelegationTokens -Dtests.method=testDelegationTokenCancelFail -Dtests.seed=5D4C2B793AE9A9E3 -Dtests.slow=true -Dtests.locale=en-GB -Dtests.timezone=Pacific/Tarawa -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] FAILURE 0.06s J0 | TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail <<<
[junit4] > Throwable #1: java.lang.AssertionError: expected:<200> but was:<404>
[junit4] > at __randomizedtesting.SeedInfo.seed([5D4C2B793AE9A9E3:35F31E53EA73BB0F]:0)
[junit4] > at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:140)
[junit4] > at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:294)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 1439098 INFO (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testZNodePaths
[junit4] 2> 1439109 INFO (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testZNodePaths
[junit4] 2> 1439141 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancel
[junit4] 2> 1439159 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1439161 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43406/solr ready
[junit4] 2> 1439163 ERROR (qtp1047257633-12178) [ ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_11]
[junit4] 2> 1439175 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1439176 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43406/solr ready
[junit4] 2> 1439193 ERROR (qtp1047257633-12167) [ ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_12]
[junit4] 2> 1439202 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancel
[junit4] 2> 1439240 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationOtherOp
[junit4] 2> 1439243 INFO (qtp2080320930-12170) [n:127.0.0.1:40185_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params op=someSolrOperation&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1439243 INFO (qtp2080320930-12170) [n:127.0.0.1:40185_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={op=someSolrOperation&action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1439244 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[5D4C2B793AE9A9E3]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationOtherOp
[junit4] 2> 1439245 INFO (jetty-launcher-1936-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@4ac53945{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 1439245 INFO (jetty-launcher-1936-thread-2) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@1dac68c6{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 1439246 INFO (jetty-launcher-1936-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=739891492
[junit4] 2> 1439246 INFO (jetty-launcher-1936-thread-2) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1563081050
[junit4] 2> 1439256 INFO (jetty-launcher-1936-thread-2) [ ] o.a.s.c.Overseer Overseer (id=97082140707586053-127.0.0.1:40185_solr-n_0000000000) closing
[junit4] 2> 1439257 INFO (OverseerStateUpdate-97082140707586053-127.0.0.1:40185_solr-n_0000000000) [n:127.0.0.1:40185_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:40185_solr
[junit4] 2> 1439257 INFO (zkCallback-1946-thread-3-processing-n:127.0.0.1:40185_solr) [n:127.0.0.1:40185_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 1439260 WARN (zkCallback-1946-thread-3-processing-n:127.0.0.1:40185_solr) [n:127.0.0.1:40185_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1439260 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x158e7b14ee90005, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 1439261 ERROR (Thread[Thread-2887,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:40185_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 1439273 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x158e7b14ee90008, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 1439274 INFO (jetty-launcher-1936-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7159a6e1{/solr,null,UNAVAILABLE}
[junit4] 2> 1440756 WARN (zkCallback-1947-thread-1-processing-n:127.0.0.1:34833_solr) [n:127.0.0.1:34833_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1440757 ERROR (Thread[Thread-2889,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:34833_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 1440761 INFO (jetty-launcher-1936-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6a63c7fc{/solr,null,UNAVAILABLE}
[junit4] 2> 1440765 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[5D4C2B793AE9A9E3]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:43406 43406
[junit4] 2> 1440858 INFO (Thread-2870) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:43406 43406
[junit4] 2> 1440860 WARN (Thread-2870) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 12 /solr/aliases.json
[junit4] 2> 12 /solr/clusterprops.json
[junit4] 2> 2 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 12 /solr/clusterstate.json
[junit4] 2> 3 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_7
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_6
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_8
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_1
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_2
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_5
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_4
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_12
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_11
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_10
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 12 /solr/live_nodes
[junit4] 2> 12 /solr/collections
[junit4] 2> 3 /solr/overseer/queue
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
[junit4] 2>
[junit4] 2> 1440861 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[5D4C2B793AE9A9E3]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_5D4C2B793AE9A9E3-001
[junit4] 2> Dec 10, 2016 7:45:09 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true): {}, locale=en-GB, timezone=Pacific/Tarawa
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=1,free=117523960,total=363036672
[junit4] 2> NOTE: All tests run in this JVM: [RequiredFieldsTest, OverseerTest, DeleteLastCustomShardedReplicaTest, CustomCollectionTest, SyncSliceTest, TestElisionMultitermQuery, SolrCLIZkUtilsTest, HdfsChaosMonkeyNothingIsSafeTest, TestOnReconnectListenerSupport, TestPostingsSolrHighlighter, HardAutoCommitTest, StatelessScriptUpdateProcessorFactoryTest, DeleteStatusTest, TestInitQParser, ParsingFieldUpdateProcessorsTest, BufferStoreTest, NumericFieldsTest, SampleTest, TestFieldCacheWithThreads, TestManagedResource, MultiTermTest, TestRequestStatusCollectionAPI, TestCSVLoader, TestCSVResponseWriter, TestSolrCloudSnapshots, EnumFieldTest, TestSolrJ, TestRandomRequestDistribution, ClassificationUpdateProcessorTest, TestReplicationHandlerBackup, TestLRUCache, SSLMigrationTest, TestConfigSetsAPIExclusivity, TestUpdate, DistributedSpellCheckComponentTest, TestUninvertingReader, TestSlowCompositeReaderWrapper, DirectUpdateHandlerTest, TestQuerySenderNoQuery, TestTestInjection, JSONWriterTest, RecoveryZkTest, TestGroupingSearch, DeleteReplicaTest, TestQueryWrapperFilter, ReplaceNodeTest, DirectSolrSpellCheckerTest, DistributedQueueTest, TestJsonFacetRefinement, TestCharFilters, TolerantUpdateProcessorTest, HdfsRestartWhileUpdatingTest, DistributedQueryComponentCustomSortTest, RulesTest, TestMacroExpander, TestSubQueryTransformerCrossCore, HdfsRecoverLeaseTest, TestWordDelimiterFilterFactory, SimpleMLTQParserTest, BooleanFieldTest, LeaderFailoverAfterPartitionTest, PeerSyncTest, DocValuesTest, DateFieldTest, TestFieldTypeResource, CoreSorterTest, BasicDistributedZkTest, DistributedMLTComponentTest, TestSmileRequest, TestUseDocValuesAsStored2, TestBlendedInfixSuggestions, DistributedQueryElevationComponentTest, InfoHandlerTest, StressHdfsTest, DistributedSuggestComponentTest, ClassificationUpdateProcessorFactoryTest, HdfsLockFactoryTest, TestCorePropertiesReload, TestBinaryField, OverseerCollectionConfigSetProcessorTest, TestMinMaxOnMultiValuedField, TestBM25SimilarityFactory, PingRequestHandlerTest, TestNumericTerms32, TemplateUpdateProcessorTest, TestCollationFieldDocValues, TestSearcherReuse, TestManagedSynonymFilterFactory, DateRangeFieldTest, TimerUtilsTest, TestSchemaResource, TestSort, DocValuesNotIndexedTest, TestSolrDynamicMBean, SortSpecParsingTest, ShardRoutingTest, TestPseudoReturnFields, TestPartialUpdateDeduplication, MergeStrategyTest, HttpPartitionTest, SolrIndexConfigTest, TestStressUserVersions, ZkStateReaderTest, TestAuthorizationFramework, PluginInfoTest, TestSolrConfigHandlerConcurrent, TestSurroundQueryParser, TestRuleBasedAuthorizationPlugin, TestLFUCache, TestTolerantSearch, LeaderElectionTest, SolrXmlInZkTest, ConcurrentDeleteAndCreateCollectionTest, RecoveryAfterSoftCommitTest, SimplePostToolTest, TestFreeTextSuggestions, ChaosMonkeyNothingIsSafeTest, TestCryptoKeys, TestBinaryResponseWriter, OverseerRolesTest, TestFieldCacheSanityChecker, TestExtendedDismaxParser, TestJettySolrRunner, TestCoreContainer, TestDynamicFieldResource, PreAnalyzedFieldManagedSchemaCloudTest, TestShortCircuitedRequests, TestChildDocTransformer, HighlighterMaxOffsetTest, TestFieldCacheSortRandom, TestCloudPivotFacet, SystemInfoHandlerTest, TestCollectionAPI, TestLeaderElectionWithEmptyReplica, TestSolrFieldCacheMBean, TestExportWriter, TestTolerantUpdateProcessorRandomCloud, TestLazyCores, TestPhraseSuggestions, TestSolrCloudWithDelegationTokens]
[junit4] Completed [278/659 (1!)] on J0 in 3.34s, 8 tests, 1 failure <<< FAILURES!
[...truncated 63632 lines...]
[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build #
1002 - Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1002/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC
3 tests failed.
FAILED: org.apache.solr.cloud.TestReplicaProperties.test
Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:49653 within 30000 ms
Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:49653 within 30000 ms
at __randomizedtesting.SeedInfo.seed([BDAF4BECC2C17A1B:35FB74366C3D17E3]:0)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:182)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:111)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:98)
at org.apache.solr.cloud.AbstractDistribZkTestBase.printLayout(AbstractDistribZkTestBase.java:322)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.distribTearDown(AbstractFullDistribZkTestBase.java:1500)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:962)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:49653 within 30000 ms
at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:233)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:174)
... 37 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestReplicaProperties
Error Message:
86 threads leaked from SUITE scope at org.apache.solr.cloud.TestReplicaProperties: 1) Thread[id=22641, name=qtp1934749681-22641, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=22777, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=22730, name=qtp1810847727-22730-acceptor-0@14e28e58-ServerConnector@30790013{HTTP/1.1,[http/1.1]}{127.0.0.1:56575}, state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=22637, name=qtp1934749681-22637, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=22698, name=searcherExecutor-10325-thread-1, state=WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=22736, name=qtp1810847727-22736, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=22639, name=qtp1934749681-22639, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=22668, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=22690, name=TEST-TestReplicaProperties.test-seed#[BDAF4BECC2C17A1B]-SendThread(127.0.0.1:49653), state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) 10) Thread[id=22766, name=qtp1507746050-22766, state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 11) Thread[id=22644, name=org.eclipse.jetty.server.session.HashSessionManager@cf286d6Timer, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 12) Thread[id=22767, name=qtp1507746050-22767, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 13) Thread[id=22660, name=Thread-5813, state=WAITING, group=TGRP-TestReplicaProperties] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1300) 14) Thread[id=22746, name=TEST-TestReplicaProperties.test-seed#[BDAF4BECC2C17A1B]-SendThread(127.0.0.1:49653), state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) 15) Thread[id=22735, name=qtp1810847727-22735, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 16) Thread[id=22694, name=Thread-5822, state=WAITING, group=TGRP-TestReplicaProperties] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1300) 17) Thread[id=22708, name=qtp1284407312-22708, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 18) Thread[id=22626, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178) at java.lang.Thread.run(Thread.java:745) 19) Thread[id=22722, name=Thread-5831, state=WAITING, group=TGRP-TestReplicaProperties] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1300) 20) Thread[id=22701, name=qtp1284407312-22701, state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 21) Thread[id=22768, name=qtp1507746050-22768, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 22) Thread[id=22658, name=OverseerHdfsCoreFailoverThread-97086765668564996-127.0.0.1:37155_-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137) at java.lang.Thread.run(Thread.java:745) 23) Thread[id=22669, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 24) Thread[id=22737, name=org.eclipse.jetty.server.session.HashSessionManager@3c7c70d9Timer, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 25) Thread[id=22734, name=qtp1810847727-22734, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 26) Thread[id=22648, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 27) Thread[id=22713, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 28) Thread[id=22726, name=searcherExecutor-10336-thread-1, state=WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 29) Thread[id=22762, name=Scheduler-1375824324, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 30) Thread[id=22680, name=qtp952978077-22680, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 31) Thread[id=22686, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 32) Thread[id=22769, name=qtp1507746050-22769, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 33) Thread[id=22625, name=Thread-5804, state=WAITING, group=TGRP-TestReplicaProperties] at java.lang.Object.wait(Native Method) at java.lang.Thread.join(Thread.java:1249) at java.lang.Thread.join(Thread.java:1323) at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302) at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309) at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490) 34) Thread[id=22764, name=qtp1507746050-22764, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 35) Thread[id=22677, name=qtp952978077-22677, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 36) Thread[id=22643, name=qtp1934749681-22643, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 37) Thread[id=22704, name=qtp1284407312-22704, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 38) Thread[id=22656, name=OverseerStateUpdate-97086765668564996-127.0.0.1:37155_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:138) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:179) at java.lang.Thread.run(Thread.java:745) 39) Thread[id=22640, name=qtp1934749681-22640, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 40) Thread[id=22707, name=qtp1284407312-22707, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 41) Thread[id=22765, name=qtp1507746050-22765-acceptor-0@2648993f-ServerConnector@11676f7d{HTTP/1.1,[http/1.1]}{127.0.0.1:49653}, state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 42) Thread[id=22702, name=qtp1284407312-22702-acceptor-0@4c7cff98-ServerConnector@3eb95982{HTTP/1.1,[http/1.1]}{127.0.0.1:44185}, state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 43) Thread[id=22750, name=Thread-5840, state=WAITING, group=TGRP-TestReplicaProperties] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1300) 44) Thread[id=22731, name=qtp1810847727-22731, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 45) Thread[id=22714, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 46) Thread[id=22681, name=org.eclipse.jetty.server.session.HashSessionManager@74750b71Timer, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 47) Thread[id=22700, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 48) Thread[id=22703, name=qtp1284407312-22703, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 49) Thread[id=22718, name=TEST-TestReplicaProperties.test-seed#[BDAF4BECC2C17A1B]-SendThread(127.0.0.1:49653), state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) 50) Thread[id=22719, name=TEST-TestReplicaProperties.test-seed#[BDAF4BECC2C17A1B]-EventThread, state=WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 51) Thread[id=22747, name=TEST-TestReplicaProperties.test-seed#[BDAF4BECC2C17A1B]-EventThread, state=WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 52) Thread[id=22676, name=qtp952978077-22676, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 53) Thread[id=22729, name=qtp1810847727-22729, state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 54) Thread[id=22742, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 55) Thread[id=22653, name=TEST-TestReplicaProperties.test-seed#[BDAF4BECC2C17A1B]-SendThread(127.0.0.1:49653), state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) 56) Thread[id=22629, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120) 57) Thread[id=22763, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 58) Thread[id=22732, name=qtp1810847727-22732, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 59) Thread[id=22771, name=qtp1507746050-22771, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 60) Thread[id=22772, name=org.eclipse.jetty.server.session.HashSessionManager@25d25a49Timer, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 61) Thread[id=22733, name=qtp1810847727-22733, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 62) Thread[id=22670, name=TEST-TestReplicaProperties.test-seed#[BDAF4BECC2C17A1B]-SendThread(127.0.0.1:49653), state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) 63) Thread[id=22709, name=org.eclipse.jetty.server.session.HashSessionManager@56dc0629Timer, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 64) Thread[id=22657, name=OverseerCollectionConfigSetProcessor-97086765668564996-127.0.0.1:37155_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:343) at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:227) at java.lang.Thread.run(Thread.java:745) 65) Thread[id=22664, name=searcherExecutor-10314-thread-1, state=WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 66) Thread[id=22673, name=qtp952978077-22673, state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 67) Thread[id=22679, name=qtp952978077-22679, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 68) Thread[id=22775, name=Scheduler-930726371, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 69) Thread[id=22741, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 70) Thread[id=22627, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Object.wait(Native Method) at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146) 71) Thread[id=22671, name=TEST-TestReplicaProperties.test-seed#[BDAF4BECC2C17A1B]-EventThread, state=WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 72) Thread[id=22675, name=qtp952978077-22675, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 73) Thread[id=22754, name=searcherExecutor-10347-thread-1, state=WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 74) Thread[id=22649, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 75) Thread[id=22674, name=qtp952978077-22674-acceptor-0@22caa235-ServerConnector@6b67907e{HTTP/1.1,[http/1.1]}{127.0.0.1:43800}, state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 76) Thread[id=22728, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 77) Thread[id=22654, name=TEST-TestReplicaProperties.test-seed#[BDAF4BECC2C17A1B]-EventThread, state=WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 78) Thread[id=22685, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:745) 79) Thread[id=22691, name=TEST-TestReplicaProperties.test-seed#[BDAF4BECC2C17A1B]-EventThread, state=WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 80) Thread[id=22770, name=qtp1507746050-22770, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 81) Thread[id=22638, name=qtp1934749681-22638-acceptor-0@72e66f2-ServerConnector@1656d329{HTTP/1.1,[http/1.1]}{127.0.0.1:37155}, state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 82) Thread[id=22678, name=qtp952978077-22678, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 83) Thread[id=22642, name=qtp1934749681-22642, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 84) Thread[id=22636, name=qtp1934749681-22636, state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 85) Thread[id=22705, name=qtp1284407312-22705, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 86) Thread[id=22706, name=qtp1284407312-22706, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 86 threads leaked from SUITE scope at org.apache.solr.cloud.TestReplicaProperties:
1) Thread[id=22641, name=qtp1934749681-22641, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
2) Thread[id=22777, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:745)
3) Thread[id=22730, name=qtp1810847727-22730-acceptor-0@14e28e58-ServerConnector@30790013{HTTP/1.1,[http/1.1]}{127.0.0.1:56575}, state=RUNNABLE, group=TGRP-TestReplicaProperties]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
4) Thread[id=22637, name=qtp1934749681-22637, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
5) Thread[id=22698, name=searcherExecutor-10325-thread-1, state=WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
6) Thread[id=22736, name=qtp1810847727-22736, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
7) Thread[id=22639, name=qtp1934749681-22639, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
8) Thread[id=22668, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:745)
9) Thread[id=22690, name=TEST-TestReplicaProperties.test-seed#[BDAF4BECC2C17A1B]-SendThread(127.0.0.1:49653), state=RUNNABLE, group=TGRP-TestReplicaProperties]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
10) Thread[id=22766, name=qtp1507746050-22766, state=RUNNABLE, group=TGRP-TestReplicaProperties]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
11) Thread[id=22644, name=org.eclipse.jetty.server.session.HashSessionManager@cf286d6Timer, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
12) Thread[id=22767, name=qtp1507746050-22767, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
13) Thread[id=22660, name=Thread-5813, state=WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1300)
14) Thread[id=22746, name=TEST-TestReplicaProperties.test-seed#[BDAF4BECC2C17A1B]-SendThread(127.0.0.1:49653), state=RUNNABLE, group=TGRP-TestReplicaProperties]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
15) Thread[id=22735, name=qtp1810847727-22735, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
16) Thread[id=22694, name=Thread-5822, state=WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1300)
17) Thread[id=22708, name=qtp1284407312-22708, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
18) Thread[id=22626, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestReplicaProperties]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
at java.lang.Thread.run(Thread.java:745)
19) Thread[id=22722, name=Thread-5831, state=WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1300)
20) Thread[id=22701, name=qtp1284407312-22701, state=RUNNABLE, group=TGRP-TestReplicaProperties]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
21) Thread[id=22768, name=qtp1507746050-22768, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
22) Thread[id=22658, name=OverseerHdfsCoreFailoverThread-97086765668564996-127.0.0.1:37155_-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
at java.lang.Thread.run(Thread.java:745)
23) Thread[id=22669, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:745)
24) Thread[id=22737, name=org.eclipse.jetty.server.session.HashSessionManager@3c7c70d9Timer, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
25) Thread[id=22734, name=qtp1810847727-22734, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
26) Thread[id=22648, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:745)
27) Thread[id=22713, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:745)
28) Thread[id=22726, name=searcherExecutor-10336-thread-1, state=WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
29) Thread[id=22762, name=Scheduler-1375824324, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
30) Thread[id=22680, name=qtp952978077-22680, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
31) Thread[id=22686, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:745)
32) Thread[id=22769, name=qtp1507746050-22769, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
33) Thread[id=22625, name=Thread-5804, state=WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Object.wait(Native Method)
at java.lang.Thread.join(Thread.java:1249)
at java.lang.Thread.join(Thread.java:1323)
at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
34) Thread[id=22764, name=qtp1507746050-22764, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
35) Thread[id=22677, name=qtp952978077-22677, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
36) Thread[id=22643, name=qtp1934749681-22643, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
37) Thread[id=22704, name=qtp1284407312-22704, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
38) Thread[id=22656, name=OverseerStateUpdate-97086765668564996-127.0.0.1:37155_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
a
[...truncated too long message...]
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 17) Thread[id=22775, name=Scheduler-930726371, state=WAITING, group=TGRP-TestReplicaProperties]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 18) Thread[id=22722, name=Thread-5831, state=WAITING, group=TGRP-TestReplicaProperties]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1300)
[junit4] > 19) Thread[id=22701, name=qtp1284407312-22701, state=RUNNABLE, group=TGRP-TestReplicaProperties]
[junit4] > at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)
[junit4] > at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)
[junit4] > at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)
[junit4] > at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
[junit4] > at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
[junit4] > at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 20) Thread[id=22754, name=searcherExecutor-10347-thread-1, state=WAITING, group=TGRP-TestReplicaProperties]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 21) Thread[id=22737, name=org.eclipse.jetty.server.session.HashSessionManager@3c7c70d9Timer, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 22) Thread[id=22726, name=searcherExecutor-10336-thread-1, state=WAITING, group=TGRP-TestReplicaProperties]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 23) Thread[id=22762, name=Scheduler-1375824324, state=WAITING, group=TGRP-TestReplicaProperties]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 24) Thread[id=22636, name=qtp1934749681-22636, state=RUNNABLE, group=TGRP-TestReplicaProperties]
[junit4] > at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
[junit4] > at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
[junit4] > at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 25) Thread[id=22750, name=Thread-5840, state=WAITING, group=TGRP-TestReplicaProperties]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1300)
[junit4] > at __randomizedtesting.SeedInfo.seed([BDAF4BECC2C17A1B]:0)
[junit4] Completed [545/659 (1!)] on J1 in 124.49s, 1 test, 3 errors <<< FAILURES!
[...truncated 62747 lines...]
[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build #
1001 - Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1001/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC
4 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.core.TestLazyCores
Error Message:
ObjectTracker found 5 object(s) that were not released!!! [MockDirectoryWrapper, MockDirectoryWrapper, SolrCore, MDCAwareThreadPoolExecutor, MockDirectoryWrapper] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347) at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:433) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:858) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:792) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:868) at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:517) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347) at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:331) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:658) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:792) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:868) at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:517) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:954) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:792) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:868) at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:517) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:815) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:792) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:868) at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:517) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:66) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:690) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:792) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:868) at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:517) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 5 object(s) that were not released!!! [MockDirectoryWrapper, MockDirectoryWrapper, SolrCore, MDCAwareThreadPoolExecutor, MockDirectoryWrapper]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:433)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:858)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:792)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:868)
at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:517)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:331)
at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:658)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:792)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:868)
at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:517)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:954)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:792)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:868)
at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:517)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:815)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:792)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:868)
at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:517)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:66)
at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:690)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:792)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:868)
at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:517)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([CE1D01E22C0FF0]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
at sun.reflect.GeneratedMethodAccessor29.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
FAILED: junit.framework.TestSuite.org.apache.solr.core.TestLazyCores
Error Message:
1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 1) Thread[id=17031, name=searcherExecutor-6179-thread-1, state=WAITING, group=TGRP-TestLazyCores] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores:
1) Thread[id=17031, name=searcherExecutor-6179-thread-1, state=WAITING, group=TGRP-TestLazyCores]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([CE1D01E22C0FF0]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.core.TestLazyCores
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=17031, name=searcherExecutor-6179-thread-1, state=WAITING, group=TGRP-TestLazyCores] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=17031, name=searcherExecutor-6179-thread-1, state=WAITING, group=TGRP-TestLazyCores]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([CE1D01E22C0FF0]:0)
FAILED: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail
Error Message:
expected:<200> but was:<404>
Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
at __randomizedtesting.SeedInfo.seed([CE1D01E22C0FF0:6871282B32B61D1C]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:140)
at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:294)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 10719 lines...]
[junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_CE1D01E22C0FF0-001/init-core-data-001
[junit4] 2> 146563 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[CE1D01E22C0FF0]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 146563 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[CE1D01E22C0FF0]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_CE1D01E22C0FF0-001/tempDir-001
[junit4] 2> 146563 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[CE1D01E22C0FF0]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 146563 INFO (Thread-1498) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 146563 INFO (Thread-1498) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 146663 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[CE1D01E22C0FF0]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:60251
[junit4] 2> 146672 INFO (jetty-launcher-222-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 146672 INFO (jetty-launcher-222-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 146674 INFO (jetty-launcher-222-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@67b7ea99{/solr,null,AVAILABLE}
[junit4] 2> 146676 INFO (jetty-launcher-222-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5a6e37ab{/solr,null,AVAILABLE}
[junit4] 2> 146676 INFO (jetty-launcher-222-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4e7acf1b{HTTP/1.1,[http/1.1]}{127.0.0.1:45020}
[junit4] 2> 146676 INFO (jetty-launcher-222-thread-1) [ ] o.e.j.s.Server Started @150586ms
[junit4] 2> 146676 INFO (jetty-launcher-222-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45020}
[junit4] 2> 146676 INFO (jetty-launcher-222-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@33e3bbd6{HTTP/1.1,[http/1.1]}{127.0.0.1:49106}
[junit4] 2> 146676 INFO (jetty-launcher-222-thread-2) [ ] o.e.j.s.Server Started @150586ms
[junit4] 2> 146676 ERROR (jetty-launcher-222-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 146677 INFO (jetty-launcher-222-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 146677 INFO (jetty-launcher-222-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 146676 INFO (jetty-launcher-222-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=49106}
[junit4] 2> 146677 INFO (jetty-launcher-222-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 146677 INFO (jetty-launcher-222-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-10T17:18:55.212Z
[junit4] 2> 146678 ERROR (jetty-launcher-222-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 146678 INFO (jetty-launcher-222-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 146678 INFO (jetty-launcher-222-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 146678 INFO (jetty-launcher-222-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 146678 INFO (jetty-launcher-222-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-10T17:18:55.213Z
[junit4] 2> 146681 INFO (jetty-launcher-222-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 146681 INFO (jetty-launcher-222-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 146692 INFO (jetty-launcher-222-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60251/solr
[junit4] 2> 146695 INFO (jetty-launcher-222-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60251/solr
[junit4] 2> 146734 INFO (jetty-launcher-222-thread-2) [n:127.0.0.1:49106_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:49106_solr
[junit4] 2> 146735 INFO (jetty-launcher-222-thread-1) [n:127.0.0.1:45020_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45020_solr
[junit4] 2> 146737 INFO (jetty-launcher-222-thread-2) [n:127.0.0.1:49106_solr ] o.a.s.c.Overseer Overseer (id=97084397001113606-127.0.0.1:49106_solr-n_0000000000) starting
[junit4] 2> 146739 INFO (zkCallback-233-thread-1-processing-n:127.0.0.1:49106_solr) [n:127.0.0.1:49106_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 146739 INFO (zkCallback-232-thread-1-processing-n:127.0.0.1:45020_solr) [n:127.0.0.1:45020_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 146750 INFO (jetty-launcher-222-thread-2) [n:127.0.0.1:49106_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49106_solr
[junit4] 2> 146752 INFO (zkCallback-232-thread-1-processing-n:127.0.0.1:45020_solr) [n:127.0.0.1:45020_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 146759 INFO (zkCallback-233-thread-1-processing-n:127.0.0.1:49106_solr) [n:127.0.0.1:49106_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 146844 INFO (jetty-launcher-222-thread-2) [n:127.0.0.1:49106_solr ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 146844 INFO (jetty-launcher-222-thread-2) [n:127.0.0.1:49106_solr ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
[junit4] 2> 146844 INFO (jetty-launcher-222-thread-2) [n:127.0.0.1:49106_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 146847 INFO (jetty-launcher-222-thread-2-EventThread) [n:127.0.0.1:49106_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 146868 INFO (jetty-launcher-222-thread-2) [n:127.0.0.1:49106_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_CE1D01E22C0FF0-001/tempDir-001/node2/.
[junit4] 2> 146903 INFO (jetty-launcher-222-thread-1) [n:127.0.0.1:45020_solr ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 146904 INFO (jetty-launcher-222-thread-1) [n:127.0.0.1:45020_solr ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
[junit4] 2> 146904 INFO (jetty-launcher-222-thread-1) [n:127.0.0.1:45020_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 146908 INFO (jetty-launcher-222-thread-1-EventThread) [n:127.0.0.1:45020_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 147000 INFO (jetty-launcher-222-thread-1) [n:127.0.0.1:45020_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_CE1D01E22C0FF0-001/tempDir-001/node1/.
[junit4] 2> 147010 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x158e9bea47c0009, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 147044 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[CE1D01E22C0FF0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenSolrClient
[junit4] 2> 147081 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147090 INFO (qtp1612945333-2623) [n:127.0.0.1:49106_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqZligFZDcsqZQECFB9V7Y0kmbO5RhWpKIGUDTyvwvpiB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 147090 INFO (qtp1612945333-2623) [n:127.0.0.1:49106_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqZligFZDcsqZQECFB9V7Y0kmbO5RhWpKIGUDTyvwvpiB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 147091 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[CE1D01E22C0FF0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenSolrClient
[junit4] 2> 147135 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[CE1D01E22C0FF0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancel
[junit4] 2> 147160 ERROR (qtp1612945333-2625) [ ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_2]
[junit4] 2> 147165 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147172 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 147174 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60251/solr ready
[junit4] 2> 147176 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147195 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 147197 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60251/solr ready
[junit4] 2> 147199 ERROR (qtp1612945333-2621) [ ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_3]
[junit4] 2> 147206 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147210 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147210 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[CE1D01E22C0FF0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancel
[junit4] 2> 147233 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[CE1D01E22C0FF0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancelFail
[junit4] 2> 147244 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[CE1D01E22C0FF0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancelFail
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSolrCloudWithDelegationTokens -Dtests.method=testDelegationTokenCancelFail -Dtests.seed=CE1D01E22C0FF0 -Dtests.slow=true -Dtests.locale=pl -Dtests.timezone=PRT -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 0.04s J0 | TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail <<<
[junit4] > Throwable #1: java.lang.AssertionError: expected:<200> but was:<404>
[junit4] > at __randomizedtesting.SeedInfo.seed([CE1D01E22C0FF0:6871282B32B61D1C]:0)
[junit4] > at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:140)
[junit4] > at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:294)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 147269 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
[junit4] 2> 147284 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 147286 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60251/solr ready
[junit4] 2> 147288 INFO (qtp1612945333-2623) [n:127.0.0.1:49106_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqc8igFZDcsrPAUCFOplz_L8zB9zKhMoHvlP6IGzCU8wB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 147288 INFO (qtp1612945333-2623) [n:127.0.0.1:49106_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqc8igFZDcsrPAUCFOplz_L8zB9zKhMoHvlP6IGzCU8wB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 147292 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147294 INFO (qtp1094523782-2612) [n:127.0.0.1:45020_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqc8igFZDcsrPAUCFOplz_L8zB9zKhMoHvlP6IGzCU8wB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 147295 INFO (qtp1094523782-2612) [n:127.0.0.1:45020_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqc8igFZDcsrPAUCFOplz_L8zB9zKhMoHvlP6IGzCU8wB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 147300 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 147301 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60251/solr ready
[junit4] 2> 147303 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147319 INFO (qtp1612945333-2624) [n:127.0.0.1:49106_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqdgigFZDcsrYAYCFOiFG3-ru3ZT3tafIO0gR9Qzv1Z5B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 147319 INFO (qtp1612945333-2624) [n:127.0.0.1:49106_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqdgigFZDcsrYAYCFOiFG3-ru3ZT3tafIO0gR9Qzv1Z5B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 147324 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 147326 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60251/solr ready
[junit4] 2> 147328 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147336 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 147337 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60251/solr ready
[junit4] 2> 147339 INFO (qtp1612945333-2623) [n:127.0.0.1:49106_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqdgigFZDcsrYAYCFOiFG3-ru3ZT3tafIO0gR9Qzv1Z5B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 147339 INFO (qtp1612945333-2623) [n:127.0.0.1:49106_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqdgigFZDcsrYAYCFOiFG3-ru3ZT3tafIO0gR9Qzv1Z5B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 147346 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 147348 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60251/solr ready
[junit4] 2> 147350 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147374 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 147375 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60251/solr ready
[junit4] 2> 147376 INFO (qtp1612945333-2624) [n:127.0.0.1:49106_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqePigFZDcsrjwcCFBwj52VNkuCCSYsNLGgGS9Pax5qhB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 147377 INFO (qtp1612945333-2624) [n:127.0.0.1:49106_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqePigFZDcsrjwcCFBwj52VNkuCCSYsNLGgGS9Pax5qhB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 147378 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x158e9bea47c0011, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 147383 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 147384 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60251/solr ready
[junit4] 2> 147386 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147389 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x158e9bea47c0012, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 147395 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 147397 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60251/solr ready
[junit4] 2> 147399 INFO (qtp1612945333-2623) [n:127.0.0.1:49106_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqePigFZDcsrjwcCFBwj52VNkuCCSYsNLGgGS9Pax5qhB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 147399 INFO (qtp1612945333-2623) [n:127.0.0.1:49106_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqePigFZDcsrjwcCFBwj52VNkuCCSYsNLGgGS9Pax5qhB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 147404 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147429 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 147430 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60251/solr ready
[junit4] 2> 147432 INFO (qtp1612945333-2615) [n:127.0.0.1:49106_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqfEigFZDcsrxAgCFLzwCFlEjzHdUfbKgY4DOCmPvb1HB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 147432 INFO (qtp1612945333-2615) [n:127.0.0.1:49106_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqfEigFZDcsrxAgCFLzwCFlEjzHdUfbKgY4DOCmPvb1HB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 147438 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 147439 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60251/solr ready
[junit4] 2> 147450 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147455 INFO (qtp1094523782-2613) [n:127.0.0.1:45020_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqfEigFZDcsrxAgCFLzwCFlEjzHdUfbKgY4DOCmPvb1HB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 147455 INFO (qtp1094523782-2613) [n:127.0.0.1:45020_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqfEigFZDcsrxAgCFLzwCFlEjzHdUfbKgY4DOCmPvb1HB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 147459 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147459 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CE1D01E22C0FF0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
[junit4] 2> 147483 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[CE1D01E22C0FF0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
[junit4] 2> 147490 WARN (qtp1612945333-2615) [n:127.0.0.1:49106_solr ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
[junit4] 2> 147490 WARN (qtp1612945333-2615) [n:127.0.0.1:49106_solr ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
[junit4] 2> 147495 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147496 WARN (qtp1094523782-2612) [n:127.0.0.1:45020_solr ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
[junit4] 2> 147496 WARN (qtp1094523782-2612) [n:127.0.0.1:45020_solr ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
[junit4] 2> 147496 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147505 WARN (qtp1612945333-2625) [ ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
[junit4] 2> 147505 WARN (qtp1612945333-2625) [ ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
[junit4] 2> 147509 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147511 WARN (qtp1094523782-2616) [ ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
[junit4] 2> 147511 WARN (qtp1094523782-2616) [ ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
[junit4] 2> 147518 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147518 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[CE1D01E22C0FF0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenewFail
[junit4] 2> 147545 INFO (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[CE1D01E22C0FF0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testZNodePaths
[junit4] 2> 147582 INFO (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[CE1D01E22C0FF0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testZNodePaths
[junit4] 2> 147611 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[CE1D01E22C0FF0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenVerify
[junit4] 2> 147620 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 147622 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60251/solr ready
[junit4] 2> 147624 INFO (qtp1612945333-2621) [n:127.0.0.1:49106_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqiSigFZDcsskgwCFOf8O5BwLgPec8UXGwrZ2vduDPnTB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 147624 INFO (qtp1612945333-2621) [n:127.0.0.1:49106_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqiSigFZDcsskgwCFOf8O5BwLgPec8UXGwrZ2vduDPnTB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 147629 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147632 INFO (qtp1094523782-2614) [n:127.0.0.1:45020_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqiSigFZDcsskgwCFOf8O5BwLgPec8UXGwrZ2vduDPnTB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 147632 INFO (qtp1094523782-2614) [n:127.0.0.1:45020_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVjpvqiSigFZDcsskgwCFOf8O5BwLgPec8UXGwrZ2vduDPnTB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 147635 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Cz, 01-sty-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Cz, 01-sty-1970 00:00:00 GMT
[junit4] 2> 147635 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[CE1D01E22C0FF0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenVerify
[junit4] 2> 147658 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[CE1D01E22C0FF0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationOtherOp
[junit4] 2> 147660 INFO (qtp1612945333-2621) [n:127.0.0.1:49106_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params op=someSolrOperation&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 147660 INFO (qtp1612945333-2621) [n:127.0.0.1:49106_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={op=someSolrOperation&action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 147661 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[CE1D01E22C0FF0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=test&e=1481426336195&s=/IbRUtb/48ATBq8m6iZGupU8h8U="; Path=/; Domain=127.0.0.1; Expires=N, 11-gru-2016 03:18:56 GMT; HttpOnly". Invalid 'expires' attribute: N, 11-gru-2016 03:18:56 GMT
[junit4] 2> 147661 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[CE1D01E22C0FF0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationOtherOp
[junit4] 2> 147663 INFO (jetty-launcher-222-thread-2) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@4e7acf1b{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 147664 INFO (jetty-launcher-222-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@33e3bbd6{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 147665 INFO (jetty-launcher-222-thread-2) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1016281743
[junit4] 2> 147665 INFO (jetty-launcher-222-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=2091482052
[junit4] 2> 147671 INFO (jetty-launcher-222-thread-1) [ ] o.a.s.c.Overseer Overseer (id=97084397001113606-127.0.0.1:49106_solr-n_0000000000) closing
[junit4] 2> 147672 INFO (OverseerStateUpdate-97084397001113606-127.0.0.1:49106_solr-n_0000000000) [n:127.0.0.1:49106_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:49106_solr
[junit4] 2> 147674 ERROR (Thread[Thread-1518,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:45020_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 147679 INFO (jetty-launcher-222-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@67b7ea99{/solr,null,UNAVAILABLE}
[junit4] 2> 149177 WARN (zkCallback-233-thread-1-processing-n:127.0.0.1:49106_solr) [n:127.0.0.1:49106_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 149177 ERROR (Thread[Thread-1515,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:49106_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 149182 INFO (jetty-launcher-222-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5a6e37ab{/solr,null,UNAVAILABLE}
[junit4] 2> 149183 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[CE1D01E22C0FF0]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60251 60251
[junit4] 2> 149197 INFO (Thread-1498) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60251 60251
[junit4] 2> 149198 WARN (Thread-1498) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 15 /solr/aliases.json
[junit4] 2> 15 /solr/clusterprops.json
[junit4] 2> 2 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 15 /solr/clusterstate.json
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_7
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_6
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_9
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_8
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_1
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_2
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_5
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_4
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_12
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_11
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_10
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 15 /solr/live_nodes
[junit4] 2> 15 /solr/collections
[junit4] 2> 2 /solr/overseer/queue
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
[junit4] 2>
[junit4] 2> 149199 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[CE1D01E22C0FF0]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_CE1D01E22C0FF0-001
[junit4] 2> Dec 10, 2016 5:18:57 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=false): {}, locale=pl, timezone=PRT
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=1,free=421601568,total=536870912
[junit4] 2> NOTE: All tests run in this JVM: [TestSlowCompositeReaderWrapper, TestBulkSchemaConcurrent, TestLeaderElectionWithEmptyReplica, SimpleFacetsTest, PreAnalyzedFieldManagedSchemaCloudTest, ZkControllerTest, TestSchemaResource, TestSchemaManager, QueryResultKeyTest, DocValuesNotIndexedTest, DeleteStatusTest, DisMaxRequestHandlerTest, TestQueryWrapperFilter, TestRawTransformer, TestFreeTextSuggestions, DistributedQueryElevationComponentTest, ReplicaListTransformerTest, TestHdfsUpdateLog, TestSolrJ, TestCharFilters, TimerUtilsTest, TestScoreJoinQPNoScore, TestIBSimilarityFactory, TestSmileRequest, TestRuleBasedAuthorizationPlugin, TestSolrFieldCacheMBean, TestSolrCloudWithSecureImpersonation, TimeZoneUtilsTest, DocExpirationUpdateProcessorFactoryTest, PathHierarchyTokenizerFactoryTest, DirectSolrConnectionTest, TestSolrCloudWithDelegationTokens]
[junit4] Completed [50/659 (1!)] on J0 in 3.16s, 8 tests, 1 failure <<< FAILURES!
[...truncated 1353 lines...]
[junit4] Suite: org.apache.solr.core.TestLazyCores
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/init-core-data-001
[junit4] 2> 1654520 INFO (SUITE-TestLazyCores-seed#[CE1D01E22C0FF0]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 1654521 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testLazySearch
[junit4] 2> 1654543 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.SolrCores Allocating transient cache for 4 transient cores
[junit4] 2> 1654564 INFO (coreLoadExecutor-6153-thread-1) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1654564 INFO (coreLoadExecutor-6153-thread-2) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1654589 INFO (coreLoadExecutor-6153-thread-2) [ ] o.a.s.s.IndexSchema [collection2] Schema name=tiny
[junit4] 2> 1654589 INFO (coreLoadExecutor-6153-thread-1) [ ] o.a.s.s.IndexSchema [collection1] Schema name=tiny
[junit4] 2> 1654592 INFO (coreLoadExecutor-6153-thread-3) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1654592 INFO (coreLoadExecutor-6153-thread-1) [ ] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1654595 INFO (coreLoadExecutor-6153-thread-1) [ ] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-001/collection1
[junit4] 2> 1654595 INFO (coreLoadExecutor-6153-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-001/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-001/collection1/data/]
[junit4] 2> 1654596 INFO (coreLoadExecutor-6153-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5794747768530669]
[junit4] 2> 1654599 INFO (coreLoadExecutor-6153-thread-3) [ ] o.a.s.s.IndexSchema [collection5] Schema name=tiny
[junit4] 2> 1654602 INFO (coreLoadExecutor-6153-thread-3) [ ] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1654603 INFO (coreLoadExecutor-6153-thread-3) [ ] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-001/collection5
[junit4] 2> 1654603 INFO (coreLoadExecutor-6153-thread-3) [ x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-001/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-001/collection5/data/]
[junit4] 2> 1654604 INFO (coreLoadExecutor-6153-thread-3) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5794747768530669]
[junit4] 2> 1654607 INFO (coreLoadExecutor-6153-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1654607 INFO (coreLoadExecutor-6153-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1654611 INFO (coreLoadExecutor-6153-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=84.1455078125, floorSegmentMB=1.0986328125, forceMergeDeletesPctAllowed=1.8750330145486216, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5423337188511159
[junit4] 2> 1654611 INFO (coreLoadExecutor-6153-thread-3) [ x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1654611 INFO (coreLoadExecutor-6153-thread-3) [ x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1654611 INFO (coreLoadExecutor-6153-thread-3) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=84.1455078125, floorSegmentMB=1.0986328125, forceMergeDeletesPctAllowed=1.8750330145486216, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5423337188511159
[junit4] 2> 1654611 INFO (coreLoadExecutor-6153-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@52ab12a5[collection1] main]
[junit4] 2> 1654611 INFO (coreLoadExecutor-6153-thread-3) [ x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@602febc8[collection5] main]
[junit4] 2> 1654611 INFO (coreLoadExecutor-6153-thread-3) [ x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-001/collection5/conf
[junit4] 2> 1654611 INFO (coreLoadExecutor-6153-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-001/collection1/conf
[junit4] 2> 1654612 INFO (coreLoadExecutor-6153-thread-3) [ x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1654612 INFO (coreLoadExecutor-6153-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1654612 INFO (searcherExecutor-6154-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@52ab12a5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1654612 INFO (searcherExecutor-6155-thread-1-processing-x:collection5) [ x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@602febc8[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1654618 INFO (coreLoadExecutor-6153-thread-2) [ ] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1654619 INFO (coreLoadExecutor-6153-thread-2) [ ] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-001/collection2
[junit4] 2> 1654620 INFO (coreLoadExecutor-6153-thread-2) [ x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-001/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-001/collection2/data/]
[junit4] 2> 1654620 INFO (coreLoadExecutor-6153-thread-2) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5794747768530669]
[junit4] 2> 1654628 INFO (coreLoadExecutor-6153-thread-2) [ x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1654628 INFO (coreLoadExecutor-6153-thread-2) [ x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1654629 INFO (coreLoadExecutor-6153-thread-2) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=84.1455078125, floorSegmentMB=1.0986328125, forceMergeDeletesPctAllowed=1.8750330145486216, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5423337188511159
[junit4] 2> 1654629 INFO (coreLoadExecutor-6153-thread-2) [ x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@4619639d[collection2] main]
[junit4] 2> 1654629 INFO (coreLoadExecutor-6153-thread-2) [ x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-001/collection2/conf
[junit4] 2> 1654629 INFO (coreLoadExecutor-6153-thread-2) [ x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1654629 INFO (searcherExecutor-6162-thread-1-processing-x:collection2) [ x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@4619639d[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1654630 INFO (coreLoadExecutor-6153-thread-2) [ x:collection2] o.a.s.c.SolrCores Opening transient core collection2
[junit4] 2> 1654637 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1654647 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ ] o.a.s.s.IndexSchema [collection4] Schema name=tiny
[junit4] 2> 1654649 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ ] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1654650 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ ] o.a.s.c.CoreContainer Creating SolrCore 'collection4' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-001/collection4
[junit4] 2> 1654651 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.c.SolrCore [[collection4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-001/collection4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-001/collection4/data/]
[junit4] 2> 1654652 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=27, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 1654659 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1654659 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1654659 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=50.70703125, floorSegmentMB=1.4794921875, forceMergeDeletesPctAllowed=15.849105699686323, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 1654660 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@3e2f1317[collection4] main]
[junit4] 2> 1654660 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-001/collection4/conf
[junit4] 2> 1654660 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1654660 INFO (searcherExecutor-6166-thread-1-processing-x:collection4) [ x:collection4] o.a.s.c.SolrCore [collection4] Registered new searcher Searcher@3e2f1317[collection4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1654661 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1654661 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@2525ae1d
[junit4] 2> 1654664 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@4532d68c[collection4] main]
[junit4] 2> 1654664 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1654664 INFO (searcherExecutor-6166-thread-1-processing-x:collection4) [ x:collection4] o.a.s.c.SolrCore [collection4] Registered new searcher Searcher@4532d68c[collection4] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):C7)))}
[junit4] 2> 1654665 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.c.S.Request [collection4] webapp=null path=null params={q={!prefix+f%3Dv_t}hel&wt=xml} hits=2 status=0 QTime=0
[junit4] 2> 1654668 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.c.S.Request [collection4] webapp=null path=null params={q={!raw+f%3Dv_t}hello&wt=xml} hits=2 status=0 QTime=0
[junit4] 2> 1654668 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.c.S.Request [collection4] webapp=null path=null params={q={!raw+f%3Dv_t}Hello&wt=xml} hits=0 status=0 QTime=0
[junit4] 2> 1654668 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.c.S.Request [collection4] webapp=null path=null params={q={!raw+f%3Dv_f}1.5&wt=xml} hits=0 status=0 QTime=0
[junit4] 2> 1654669 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.c.S.Request [collection1] webapp=null path=null params={q={!raw+f%3Dv_t}hello&wt=xml} hits=0 status=0 QTime=0
[junit4] 2> 1654669 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.c.CoreContainer Shutting down CoreContainer instance=443457003
[junit4] 2> 1654672 INFO (coreCloseExecutor-6170-thread-1-processing-x:collection4) [ x:collection5] o.a.s.c.SolrCore [collection5] CLOSING SolrCore org.apache.solr.core.SolrCore@14161e37
[junit4] 2> 1654673 INFO (coreCloseExecutor-6170-thread-4-processing-x:collection4) [ x:collection2] o.a.s.c.SolrCore [collection2] CLOSING SolrCore org.apache.solr.core.SolrCore@7ab8392
[junit4] 2> 1654684 INFO (coreCloseExecutor-6170-thread-3-processing-x:collection4) [ x:collection4] o.a.s.c.SolrCore [collection4] CLOSING SolrCore org.apache.solr.core.SolrCore@51da20c3
[junit4] 2> 1654695 INFO (coreCloseExecutor-6170-thread-2-processing-x:collection4) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@2747d547
[junit4] 2> 1654696 INFO (TEST-TestLazyCores.testLazySearch-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.SolrTestCaseJ4 ###Ending testLazySearch
[junit4] 2> 1654697 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.SolrTestCaseJ4 ###Starting testCachingLimit
[junit4] 2> 1654719 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CE1D01E22C0FF0]) [ x:collection4] o.a.s.c.SolrCores Allocating transient cache for 4 transient cores
[junit4] 2> 1654746 INFO (coreLoadExecutor-6177-thread-2-processing-x:collection4) [ x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1654746 INFO (coreLoadExecutor-6177-thread-3-processing-x:collection4) [ x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1654746 INFO (coreLoadExecutor-6177-thread-1-processing-x:collection4) [ x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1654770 INFO (coreLoadExecutor-6177-thread-3-processing-x:collection4) [ x:collection4] o.a.s.s.IndexSchema [collection5] Schema name=tiny
[junit4] 2> 1654770 INFO (coreLoadExecutor-6177-thread-1-processing-x:collection4) [ x:collection4] o.a.s.s.IndexSchema [collection1] Schema name=tiny
[junit4] 2> 1654775 INFO (coreLoadExecutor-6177-thread-1-processing-x:collection4) [ x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1654776 INFO (coreLoadExecutor-6177-thread-1-processing-x:collection4) [ x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-002/collection1
[junit4] 2> 1654777 INFO (coreLoadExecutor-6177-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-002/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-002/collection1/data/]
[junit4] 2> 1654777 INFO (coreLoadExecutor-6177-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5794747768530669]
[junit4] 2> 1654785 INFO (coreLoadExecutor-6177-thread-2-processing-x:collection4) [ x:collection4] o.a.s.s.IndexSchema [collection2] Schema name=tiny
[junit4] 2> 1654788 INFO (coreLoadExecutor-6177-thread-2-processing-x:collection4) [ x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1654789 INFO (coreLoadExecutor-6177-thread-2-processing-x:collection4) [ x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-002/collection2
[junit4] 2> 1654789 INFO (coreLoadExecutor-6177-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-002/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-002/collection2/data/]
[junit4] 2> 1654795 INFO (coreLoadExecutor-6177-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1654795 INFO (coreLoadExecutor-6177-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1654795 INFO (coreLoadExecutor-6177-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=84.1455078125, floorSegmentMB=1.0986328125, forceMergeDeletesPctAllowed=1.8750330145486216, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5423337188511159
[junit4] 2> 1654795 INFO (coreLoadExecutor-6177-thread-1-processing-x:collection4) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6745a2cd[collection1] main]
[junit4] 2> 1654795 INFO (coreLoadExecutor-6177-thread-1-processing-x:collection4) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-002/collection1/conf
[junit4] 2> 1654795 INFO (coreLoadExecutor-6177-thread-1-processing-x:collection4) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1654796 INFO (searcherExecutor-6178-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6745a2cd[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1654796 INFO (coreLoadExecutor-6177-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5794747768530669]
[junit4] 2> 1654818 INFO (coreLoadExecutor-6177-thread-3-processing-x:collection4) [ x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1654830 INFO (coreLoadExecutor-6177-thread-3-processing-x:collection4) [ x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-002/collection5
[junit4] 2> 1654831 INFO (coreLoadExecutor-6177-thread-3-processing-x:collection4) [ x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-002/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-002/collection5/data/]
[junit4] 2> 1654859 INFO (coreLoadExecutor-6177-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5794747768530669]
[junit4] 2> 1654875 INFO (coreLoadExecutor-6177-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1654875 INFO (coreLoadExecutor-6177-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1654875 INFO (coreLoadExecutor-6177-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=84.1455078125, floorSegmentMB=1.0986328125, forceMergeDeletesPctAllowed=1.8750330145486216, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5423337188511159
[junit4] 2> 1654875 INFO (coreLoadExecutor-6177-thread-2-processing-x:collection4) [ x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@39ae018d[collection2] main]
[junit4] 2> 1654876 INFO (coreLoadExecutor-6177-thread-2-processing-x:collection4) [ x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_CE1D01E22C0FF0-001/tempDir-002/collection2/conf
[junit4] 2> 1654876 INFO (coreLoadExecutor-6177-thread-2-processing-x:collection4) [ x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1654876 INFO (coreLoadExecutor-6177-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.SolrCores Opening transient core collection2
[junit4] 2> 1654876 INFO (searcherExecutor-6179-thread-1-processing-x:collection2) [ x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@39ae018d[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1654889 INFO (coreLoadExecutor-6177-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1654890 INFO (coreLoadExecutor-6177-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1654890 INFO (coreLoadExecutor-6177-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=84.1455078125, floorSegmentMB=1.0986328125, forceMergeDeletesPctAllowed=1.8750330145486216, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5423337188511159
[junit4] 2> 1654890 INFO (coreLoadExecutor-6177-thread-3-processing-x:collection4) [ x:collection5] o.a.s.s.SolrIndexSearcher O
[...truncated too long message...]
PrimUtilsTest, DeleteReplicaTest, ZkNodePropsTest, TestXIncludeConfig, HdfsChaosMonkeyNothingIsSafeTest, TestHighlightDedupGrouping, TestUseDocValuesAsStored2, TestFastLRUCache, DistributedQueryComponentCustomSortTest, TestSortByMinMaxFunction, TestSearcherReuse, TestTolerantSearch, TestGroupingSearch, DirectUpdateHandlerTest, TestSubQueryTransformerCrossCore, OverseerCollectionConfigSetProcessorTest, MBeansHandlerTest, FastVectorHighlighterTest, CustomCollectionTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestConfigSetProperties, DateRangeFieldTest, TestMaxScoreQueryParser, TestReplicaProperties, SyncSliceTest, AlternateDirectoryTest, TestBinaryResponseWriter, TestCloudRecovery, BlockJoinFacetDistribTest, TestDistributedSearch, TestPseudoReturnFields, BlockCacheTest, StressHdfsTest, TestGraphMLResponseWriter, BaseCdcrDistributedZkTest, MoreLikeThisHandlerTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, OverseerTest, TestReplicationHandler, ZkCLITest, DistributedSpellCheckComponentTest, TestStressReorder, TestReload, DistributedTermsComponentTest, SolrCmdDistributorTest, BadIndexSchemaTest, TestSort, TestLazyCores]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestLazyCores -Dtests.seed=CE1D01E22C0FF0 -Dtests.slow=true -Dtests.locale=lt-LT -Dtests.timezone=Antarctica/Palmer -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J0 | TestLazyCores (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker found 5 object(s) that were not released!!! [MockDirectoryWrapper, MockDirectoryWrapper, SolrCore, MDCAwareThreadPoolExecutor, MockDirectoryWrapper]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
[junit4] > at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:433)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:858)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:792)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:868)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:517)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
[junit4] > at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:331)
[junit4] > at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:658)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:792)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:868)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:517)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:954)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:792)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:868)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:517)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:815)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:792)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:868)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:517)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
[junit4] > at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:66)
[junit4] > at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:690)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:864)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:792)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:868)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:517)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([CE1D01E22C0FF0]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
[junit4] > at java.lang.Thread.run(Thread.java:745)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores:
[junit4] > 1) Thread[id=17031, name=searcherExecutor-6179-thread-1, state=WAITING, group=TGRP-TestLazyCores]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([CE1D01E22C0FF0]:0)Throwable #3: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=17031, name=searcherExecutor-6179-thread-1, state=WAITING, group=TGRP-TestLazyCores]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([CE1D01E22C0FF0]:0)
[junit4] Completed [465/659 (2!)] on J0 in 181.88s, 8 tests, 1 failure, 2 errors <<< FAILURES!
[...truncated 63002 lines...]