You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/10/15 20:50:58 UTC
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 243 -
Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/243/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC
1 tests failed.
FAILED: org.apache.solr.cloud.TestPullReplica.testKillLeader
Error Message:
Replica state not updated in cluster state null Live Nodes: [127.0.0.1:46484_solr, 127.0.0.1:53709_solr] Last available state: DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/5)={ "pullReplicas":"1", "replicationFactor":"1", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{ "core_node3":{ "core":"pull_replica_test_kill_leader_shard1_replica_n1", "base_url":"http://127.0.0.1:53709/solr", "node_name":"127.0.0.1:53709_solr", "state":"down", "type":"NRT", "leader":"true"}, "core_node4":{ "core":"pull_replica_test_kill_leader_shard1_replica_p2", "base_url":"http://127.0.0.1:46484/solr", "node_name":"127.0.0.1:46484_solr", "state":"active", "type":"PULL"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"100", "autoAddReplicas":"false", "nrtReplicas":"1", "tlogReplicas":"0"}
Stack Trace:
java.lang.AssertionError: Replica state not updated in cluster state
null
Live Nodes: [127.0.0.1:46484_solr, 127.0.0.1:53709_solr]
Last available state: DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/5)={
"pullReplicas":"1",
"replicationFactor":"1",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{
"core_node3":{
"core":"pull_replica_test_kill_leader_shard1_replica_n1",
"base_url":"http://127.0.0.1:53709/solr",
"node_name":"127.0.0.1:53709_solr",
"state":"down",
"type":"NRT",
"leader":"true"},
"core_node4":{
"core":"pull_replica_test_kill_leader_shard1_replica_p2",
"base_url":"http://127.0.0.1:46484/solr",
"node_name":"127.0.0.1:46484_solr",
"state":"active",
"type":"PULL"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"100",
"autoAddReplicas":"false",
"nrtReplicas":"1",
"tlogReplicas":"0"}
at __randomizedtesting.SeedInfo.seed([65B3A3DE4297EDBD:2CA5576A202C79EB]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
at org.apache.solr.cloud.TestPullReplica.doTestNoLeader(TestPullReplica.java:401)
at org.apache.solr.cloud.TestPullReplica.testKillLeader(TestPullReplica.java:290)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12567 lines...]
[junit4] Suite: org.apache.solr.cloud.TestPullReplica
[junit4] 2> 1770823 INFO (SUITE-TestPullReplica-seed#[65B3A3DE4297EDBD]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_65B3A3DE4297EDBD-001/init-core-data-001
[junit4] 2> 1770824 WARN (SUITE-TestPullReplica-seed#[65B3A3DE4297EDBD]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=9 numCloses=9
[junit4] 2> 1770824 INFO (SUITE-TestPullReplica-seed#[65B3A3DE4297EDBD]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 1770825 INFO (SUITE-TestPullReplica-seed#[65B3A3DE4297EDBD]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 1770826 INFO (SUITE-TestPullReplica-seed#[65B3A3DE4297EDBD]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_65B3A3DE4297EDBD-001/tempDir-001
[junit4] 2> 1770826 INFO (SUITE-TestPullReplica-seed#[65B3A3DE4297EDBD]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1770826 INFO (Thread-5455) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1770827 INFO (Thread-5455) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1770829 ERROR (Thread-5455) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1770930 INFO (SUITE-TestPullReplica-seed#[65B3A3DE4297EDBD]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:47484
[junit4] 2> 1770937 INFO (jetty-launcher-2805-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1770938 INFO (jetty-launcher-2805-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1770940 INFO (jetty-launcher-2805-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@76d282d8{/solr,null,AVAILABLE}
[junit4] 2> 1770940 INFO (jetty-launcher-2805-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@64f06527{HTTP/1.1,[http/1.1]}{127.0.0.1:53709}
[junit4] 2> 1770940 INFO (jetty-launcher-2805-thread-1) [ ] o.e.j.s.Server Started @1773056ms
[junit4] 2> 1770940 INFO (jetty-launcher-2805-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=53709}
[junit4] 2> 1770941 INFO (jetty-launcher-2805-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@37d6ebdd{/solr,null,AVAILABLE}
[junit4] 2> 1770941 ERROR (jetty-launcher-2805-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1770941 INFO (jetty-launcher-2805-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 1770941 INFO (jetty-launcher-2805-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1770941 INFO (jetty-launcher-2805-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1770941 INFO (jetty-launcher-2805-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-15T19:58:08.605Z
[junit4] 2> 1770941 INFO (jetty-launcher-2805-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6679b5c2{HTTP/1.1,[http/1.1]}{127.0.0.1:46484}
[junit4] 2> 1770941 INFO (jetty-launcher-2805-thread-2) [ ] o.e.j.s.Server Started @1773056ms
[junit4] 2> 1770941 INFO (jetty-launcher-2805-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46484}
[junit4] 2> 1770941 ERROR (jetty-launcher-2805-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1770941 INFO (jetty-launcher-2805-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 1770941 INFO (jetty-launcher-2805-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1770941 INFO (jetty-launcher-2805-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1770941 INFO (jetty-launcher-2805-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-15T19:58:08.605Z
[junit4] 2> 1770948 INFO (jetty-launcher-2805-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1770948 INFO (jetty-launcher-2805-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1770955 INFO (jetty-launcher-2805-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:47484/solr
[junit4] 2> 1770957 INFO (jetty-launcher-2805-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:47484/solr
[junit4] 2> 1771137 INFO (jetty-launcher-2805-thread-2) [n:127.0.0.1:46484_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1771138 INFO (jetty-launcher-2805-thread-2) [n:127.0.0.1:46484_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46484_solr
[junit4] 2> 1771140 INFO (jetty-launcher-2805-thread-2) [n:127.0.0.1:46484_solr ] o.a.s.c.Overseer Overseer (id=98834677005811718-127.0.0.1:46484_solr-n_0000000000) starting
[junit4] 2> 1771144 INFO (jetty-launcher-2805-thread-1) [n:127.0.0.1:53709_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1771145 INFO (jetty-launcher-2805-thread-1) [n:127.0.0.1:53709_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53709_solr
[junit4] 2> 1771148 INFO (zkCallback-2816-thread-1-processing-n:127.0.0.1:53709_solr) [n:127.0.0.1:53709_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1771149 INFO (zkCallback-2817-thread-1-processing-n:127.0.0.1:46484_solr) [n:127.0.0.1:46484_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1771154 INFO (jetty-launcher-2805-thread-2) [n:127.0.0.1:46484_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46484_solr
[junit4] 2> 1771155 INFO (zkCallback-2816-thread-1-processing-n:127.0.0.1:53709_solr) [n:127.0.0.1:53709_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1771190 INFO (zkCallback-2817-thread-1-processing-n:127.0.0.1:46484_solr) [n:127.0.0.1:46484_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1771257 INFO (jetty-launcher-2805-thread-2) [n:127.0.0.1:46484_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46484.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cbb3603
[junit4] 2> 1771263 INFO (jetty-launcher-2805-thread-2) [n:127.0.0.1:46484_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46484.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cbb3603
[junit4] 2> 1771263 INFO (jetty-launcher-2805-thread-2) [n:127.0.0.1:46484_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46484.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cbb3603
[junit4] 2> 1771264 INFO (jetty-launcher-2805-thread-2) [n:127.0.0.1:46484_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_65B3A3DE4297EDBD-001/tempDir-001/node2/.
[junit4] 2> 1771508 INFO (jetty-launcher-2805-thread-1) [n:127.0.0.1:53709_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53709.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cbb3603
[junit4] 2> 1771514 INFO (jetty-launcher-2805-thread-1) [n:127.0.0.1:53709_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53709.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cbb3603
[junit4] 2> 1771514 INFO (jetty-launcher-2805-thread-1) [n:127.0.0.1:53709_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53709.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cbb3603
[junit4] 2> 1771515 INFO (jetty-launcher-2805-thread-1) [n:127.0.0.1:53709_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_65B3A3DE4297EDBD-001/tempDir-001/node1/.
[junit4] 2> 1771555 INFO (SUITE-TestPullReplica-seed#[65B3A3DE4297EDBD]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1771557 INFO (SUITE-TestPullReplica-seed#[65B3A3DE4297EDBD]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:47484/solr ready
[junit4] 2> 1771565 INFO (SUITE-TestPullReplica-seed#[65B3A3DE4297EDBD]-worker) [ ] o.a.s.c.TestPullReplica Using legacyCloud?: false
[junit4] 2> 1771571 INFO (qtp824552040-18334) [n:127.0.0.1:46484_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1771573 INFO (qtp824552040-18334) [n:127.0.0.1:46484_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1771598 INFO (TEST-TestPullReplica.testSearchWhileReplicationHappens-seed#[65B3A3DE4297EDBD]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSearchWhileReplicationHappens
[junit4] 2> 1771599 INFO (TEST-TestPullReplica.testSearchWhileReplicationHappens-seed#[65B3A3DE4297EDBD]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSearchWhileReplicationHappens
[junit4] 2> 1771622 INFO (TEST-TestPullReplica.testRealTimeGet-seed#[65B3A3DE4297EDBD]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testRealTimeGet
[junit4] 2> 1771623 INFO (qtp824552040-18335) [n:127.0.0.1:46484_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=2&replicationFactor=2&collection.configName=conf&maxShardsPerNode=100&name=pull_replica_test_real_time_get&nrtReplicas=2&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1771627 INFO (OverseerThreadFactory-6060-thread-1-processing-n:127.0.0.1:46484_solr) [n:127.0.0.1:46484_solr ] o.a.s.c.CreateCollectionCmd Create collection pull_replica_test_real_time_get
[junit4] 2> 1771628 WARN (OverseerThreadFactory-6060-thread-1-processing-n:127.0.0.1:46484_solr) [n:127.0.0.1:46484_solr ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 4 on collection pull_replica_test_real_time_get is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
[junit4] 2> 1771740 INFO (OverseerStateUpdate-98834677005811718-127.0.0.1:46484_solr-n_0000000000) [n:127.0.0.1:46484_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"pull_replica_test_real_time_get",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"pull_replica_test_real_time_get_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:46484/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 1771744 INFO (OverseerStateUpdate-98834677005811718-127.0.0.1:46484_solr-n_0000000000) [n:127.0.0.1:46484_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"pull_replica_test_real_time_get",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"pull_replica_test_real_time_get_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:53709/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 1771748 INFO (OverseerStateUpdate-98834677005811718-127.0.0.1:46484_solr-n_0000000000) [n:127.0.0.1:46484_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"pull_replica_test_real_time_get",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"pull_replica_test_real_time_get_shard1_replica_p4",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:46484/solr",
[junit4] 2> "type":"PULL"}
[junit4] 2> 1771751 INFO (OverseerStateUpdate-98834677005811718-127.0.0.1:46484_solr-n_0000000000) [n:127.0.0.1:46484_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"pull_replica_test_real_time_get",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"pull_replica_test_real_time_get_shard1_replica_p6",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:53709/solr",
[junit4] 2> "type":"PULL"}
[junit4] 2> 1771957 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_n1&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1771958 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1771960 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_n2&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1771960 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_p4&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=PULL
[junit4] 2> 1771960 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1771961 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_p6&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=PULL
[junit4] 2> 1772077 INFO (zkCallback-2817-thread-1-processing-n:127.0.0.1:46484_solr) [n:127.0.0.1:46484_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1772078 INFO (zkCallback-2816-thread-1-processing-n:127.0.0.1:53709_solr) [n:127.0.0.1:53709_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1772078 INFO (zkCallback-2817-thread-2-processing-n:127.0.0.1:46484_solr) [n:127.0.0.1:46484_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1772078 INFO (zkCallback-2816-thread-2-processing-n:127.0.0.1:53709_solr) [n:127.0.0.1:53709_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1772979 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 1772983 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 1772986 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 1772988 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 1773000 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.s.IndexSchema [pull_replica_test_real_time_get_shard1_replica_n1] Schema name=minimal
[junit4] 2> 1773000 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.s.IndexSchema [pull_replica_test_real_time_get_shard1_replica_p4] Schema name=minimal
[junit4] 2> 1773002 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.s.IndexSchema [pull_replica_test_real_time_get_shard1_replica_n2] Schema name=minimal
[junit4] 2> 1773003 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1773003 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_real_time_get_shard1_replica_p4' using configuration from collection pull_replica_test_real_time_get, trusted=true
[junit4] 2> 1773003 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.s.IndexSchema [pull_replica_test_real_time_get_shard1_replica_p6] Schema name=minimal
[junit4] 2> 1773003 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46484.solr.core.pull_replica_test_real_time_get.shard1.replica_p4' (registry 'solr.core.pull_replica_test_real_time_get.shard1.replica_p4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cbb3603
[junit4] 2> 1773003 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1773004 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.SolrCore [[pull_replica_test_real_time_get_shard1_replica_p4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_65B3A3DE4297EDBD-001/tempDir-001/node2/pull_replica_test_real_time_get_shard1_replica_p4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_65B3A3DE4297EDBD-001/tempDir-001/node2/./pull_replica_test_real_time_get_shard1_replica_p4/data/]
[junit4] 2> 1773004 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1773004 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_real_time_get_shard1_replica_n2' using configuration from collection pull_replica_test_real_time_get, trusted=true
[junit4] 2> 1773005 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53709.solr.core.pull_replica_test_real_time_get.shard1.replica_n2' (registry 'solr.core.pull_replica_test_real_time_get.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cbb3603
[junit4] 2> 1773005 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1773005 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.SolrCore [[pull_replica_test_real_time_get_shard1_replica_n2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_65B3A3DE4297EDBD-001/tempDir-001/node1/pull_replica_test_real_time_get_shard1_replica_n2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_65B3A3DE4297EDBD-001/tempDir-001/node1/./pull_replica_test_real_time_get_shard1_replica_n2/data/]
[junit4] 2> 1773006 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1773006 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_real_time_get_shard1_replica_p6' using configuration from collection pull_replica_test_real_time_get, trusted=true
[junit4] 2> 1773006 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53709.solr.core.pull_replica_test_real_time_get.shard1.replica_p6' (registry 'solr.core.pull_replica_test_real_time_get.shard1.replica_p6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cbb3603
[junit4] 2> 1773006 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1773006 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.SolrCore [[pull_replica_test_real_time_get_shard1_replica_p6] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_65B3A3DE4297EDBD-001/tempDir-001/node1/pull_replica_test_real_time_get_shard1_replica_p6], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_65B3A3DE4297EDBD-001/tempDir-001/node1/./pull_replica_test_real_time_get_shard1_replica_p6/data/]
[junit4] 2> 1773015 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1773015 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_real_time_get_shard1_replica_n1' using configuration from collection pull_replica_test_real_time_get, trusted=true
[junit4] 2> 1773016 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46484.solr.core.pull_replica_test_real_time_get.shard1.replica_n1' (registry 'solr.core.pull_replica_test_real_time_get.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cbb3603
[junit4] 2> 1773016 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1773016 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.SolrCore [[pull_replica_test_real_time_get_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_65B3A3DE4297EDBD-001/tempDir-001/node2/pull_replica_test_real_time_get_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_65B3A3DE4297EDBD-001/tempDir-001/node2/./pull_replica_test_real_time_get_shard1_replica_n1/data/]
[junit4] 2> 1773061 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1773061 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1773075 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1773075 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1773076 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1773076 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1773077 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@15ff7760[pull_replica_test_real_time_get_shard1_replica_n2] main]
[junit4] 2> 1773079 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.s.SolrIndexSearcher Opening [Searcher@144481[pull_replica_test_real_time_get_shard1_replica_p4] main]
[junit4] 2> 1773080 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 1773080 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 1773080 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1773081 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581354834455429120
[junit4] 2> 1773081 INFO (searcherExecutor-6066-thread-1-processing-n:127.0.0.1:53709_solr x:pull_replica_test_real_time_get_shard1_replica_n2 s:shard1 c:pull_replica_test_real_time_get r:core_node5) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.SolrCore [pull_replica_test_real_time_get_shard1_replica_n2] Registered new searcher Searcher@15ff7760[pull_replica_test_real_time_get_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1773082 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1773082 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1773083 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.s.SolrIndexSearcher Opening [Searcher@13d369df[pull_replica_test_real_time_get_shard1_replica_p6] main]
[junit4] 2> 1773083 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 1773084 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 1773085 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 1773085 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1773086 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.ZkController pull_replica_test_real_time_get_shard1_replica_p6 starting background replication from leader
[junit4] 2> 1773086 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 1773087 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 1773087 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 1773087 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1773087 INFO (searcherExecutor-6067-thread-1-processing-n:127.0.0.1:53709_solr x:pull_replica_test_real_time_get_shard1_replica_p6 s:shard1 c:pull_replica_test_real_time_get r:core_node8) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.SolrCore [pull_replica_test_real_time_get_shard1_replica_p6] Registered new searcher Searcher@13d369df[pull_replica_test_real_time_get_shard1_replica_p6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1773087 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 1773088 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1773089 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.ZkController pull_replica_test_real_time_get_shard1_replica_p4 starting background replication from leader
[junit4] 2> 1773089 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 1773090 INFO (searcherExecutor-6065-thread-1-processing-n:127.0.0.1:46484_solr x:pull_replica_test_real_time_get_shard1_replica_p4 s:shard1 c:pull_replica_test_real_time_get r:core_node7) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.SolrCore [pull_replica_test_real_time_get_shard1_replica_p4] Registered new searcher Searcher@144481[pull_replica_test_real_time_get_shard1_replica_p4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1773090 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 1773091 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1773105 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1773105 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1773106 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1773106 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1773107 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6f446ee9[pull_replica_test_real_time_get_shard1_replica_n1] main]
[junit4] 2> 1773109 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 1773109 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 1773110 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1773110 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581354834485837824
[junit4] 2> 1773111 INFO (searcherExecutor-6068-thread-1-processing-n:127.0.0.1:46484_solr x:pull_replica_test_real_time_get_shard1_replica_n1 s:shard1 c:pull_replica_test_real_time_get r:core_node3) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.SolrCore [pull_replica_test_real_time_get_shard1_replica_n1] Registered new searcher Searcher@6f446ee9[pull_replica_test_real_time_get_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1773191 INFO (zkCallback-2817-thread-2-processing-n:127.0.0.1:46484_solr) [n:127.0.0.1:46484_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1773191 INFO (zkCallback-2816-thread-2-processing-n:127.0.0.1:53709_solr) [n:127.0.0.1:53709_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1773191 INFO (zkCallback-2816-thread-1-processing-n:127.0.0.1:53709_solr) [n:127.0.0.1:53709_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1773191 INFO (zkCallback-2817-thread-1-processing-n:127.0.0.1:46484_solr) [n:127.0.0.1:46484_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1773588 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1773588 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1773588 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/
[junit4] 2> 1773589 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=pull_replica_test_real_time_get_shard1_replica_n2 url=http://127.0.0.1:53709/solr START replicas=[http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/] nUpdates=100
[junit4] 2> 1773590 INFO (qtp824552040-18331) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1773889 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=pull_replica_test_real_time_get_shard1_replica_n2 url=http://127.0.0.1:53709/solr DONE. We have no versions. sync failed.
[junit4] 2> 1773889 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 1773889 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 1773889 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1773895 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/ shard1
[junit4] 2> 1774000 INFO (zkCallback-2816-thread-2-processing-n:127.0.0.1:53709_solr) [n:127.0.0.1:53709_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1774000 INFO (zkCallback-2816-thread-1-processing-n:127.0.0.1:53709_solr) [n:127.0.0.1:53709_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1774005 INFO (zkCallback-2817-thread-1-processing-n:127.0.0.1:46484_solr) [n:127.0.0.1:46484_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1774005 INFO (zkCallback-2817-thread-2-processing-n:127.0.0.1:46484_solr) [n:127.0.0.1:46484_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1774038 INFO (indexFetcher-6082-thread-1) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.h.IndexFetcher Replica core_node5 is leader but it's state is down, skipping replication
[junit4] 2> 1774043 INFO (indexFetcher-6080-thread-1) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.h.IndexFetcher Replica core_node5 is leader but it's state is down, skipping replication
[junit4] 2> 1774046 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1774048 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_n2&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2088
[junit4] 2> 1774091 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_p6&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=PULL} status=0 QTime=2131
[junit4] 2> 1774093 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_p4&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=PULL} status=0 QTime=2134
[junit4] 2> 1774117 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_n1&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2160
[junit4] 2> 1774120 INFO (qtp824552040-18335) [n:127.0.0.1:46484_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1774221 INFO (zkCallback-2817-thread-2-processing-n:127.0.0.1:46484_solr) [n:127.0.0.1:46484_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1774221 INFO (zkCallback-2817-thread-1-processing-n:127.0.0.1:46484_solr) [n:127.0.0.1:46484_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1774221 INFO (zkCallback-2816-thread-2-processing-n:127.0.0.1:53709_solr) [n:127.0.0.1:53709_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1774221 INFO (zkCallback-2816-thread-1-processing-n:127.0.0.1:53709_solr) [n:127.0.0.1:53709_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1775121 INFO (qtp824552040-18335) [n:127.0.0.1:46484_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=2&replicationFactor=2&collection.configName=conf&maxShardsPerNode=100&name=pull_replica_test_real_time_get&nrtReplicas=2&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2} status=0 QTime=3499
[junit4] 2> 1775134 INFO (qtp824552040-18334) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/&wt=javabin&version=2}{add=[0 (1581354836602912768)]} 0 1
[junit4] 2> 1775134 INFO (qtp873174652-18330) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_real_time_get_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/&wt=javabin&version=2}{add=[0 (1581354836602912768)]} 0 5
[junit4] 2> 1775135 INFO (qtp824552040-18339) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[0]} 0 7
[junit4] 2> 1775137 INFO (qtp873174652-18332) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/|http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/
[junit4] 2> 1775138 INFO (qtp824552040-18331) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/get params={distrib=false&qt=/get&_stateVer_=pull_replica_test_real_time_get:5&omitHeader=true&shards.purpose=1&NOW=1508097492801&ids=0&isShard=true&shard.url=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/|http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 1775139 INFO (qtp873174652-18332) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_p6] webapp=/solr path=/get params={qt=/get&_stateVer_=pull_replica_test_real_time_get:5&ids=0&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1775139 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/get params={qt=/get&ids=0&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1775140 INFO (qtp873174652-18328) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n2] webapp=/solr path=/get params={qt=/get&ids=0&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1775141 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/|http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/
[junit4] 2> 1775142 INFO (qtp824552040-18335) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/get params={distrib=false&qt=/get&omitHeader=true&shards.purpose=1&NOW=1508097492805&ids=0&isShard=true&shard.url=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/|http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 1775143 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_p4] webapp=/solr path=/get params={qt=/get&ids=0&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 1775144 INFO (qtp873174652-18328) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/|http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/
[junit4] 2> 1775145 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n2] webapp=/solr path=/get params={distrib=false&qt=/get&omitHeader=true&shards.purpose=1&NOW=1508097492807&ids=0&isShard=true&shard.url=http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/|http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 1775145 INFO (qtp873174652-18328) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_p6] webapp=/solr path=/get params={qt=/get&ids=0&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 1775147 INFO (qtp824552040-18334) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/&wt=javabin&version=2}{add=[1 (1581354836620738560)]} 0 0
[junit4] 2> 1775147 INFO (qtp873174652-18327) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_real_time_get_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1 (1581354836620738560)]} 0 1
[junit4] 2> 1775148 INFO (qtp824552040-18339) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/|http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/
[junit4] 2> 1775149 INFO (qtp873174652-18330) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n2] webapp=/solr path=/get params={distrib=false&qt=/get&_stateVer_=pull_replica_test_real_time_get:5&omitHeader=true&shards.purpose=1&NOW=1508097492812&ids=1&isShard=true&shard.url=http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/|http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 1775149 INFO (qtp824552040-18339) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_p4] webapp=/solr path=/get params={qt=/get&_stateVer_=pull_replica_test_real_time_get:5&ids=1&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 1775149 INFO (qtp824552040-18338) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/get params={qt=/get&ids=1&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1775149 INFO (qtp873174652-18329) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n2] webapp=/solr path=/get params={qt=/get&ids=1&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1775154 INFO (qtp824552040-18331) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/|http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/
[junit4] 2> 1775155 INFO (qtp873174652-18332) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n2] webapp=/solr path=/get params={distrib=false&qt=/get&omitHeader=true&shards.purpose=1&NOW=1508097492818&ids=1&isShard=true&shard.url=http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/|http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 1775156 INFO (qtp824552040-18331) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_p4] webapp=/solr path=/get params={qt=/get&ids=1&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 1775157 INFO (qtp873174652-18332) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/|http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/
[junit4] 2> 1775157 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n2] webapp=/solr path=/get params={distrib=false&qt=/get&omitHeader=true&shards.purpose=1&NOW=1508097492820&ids=1&isShard=true&shard.url=http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/|http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 1775159 INFO (qtp873174652-18332) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_p6] webapp=/solr path=/get params={qt=/get&ids=1&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1775162 INFO (qtp824552040-18335) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/&wt=javabin&version=2}{add=[2 (1581354836636467200)]} 0 0
[junit4] 2> 1775162 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_real_time_get_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_p4/&wt=javabin&version=2}{add=[2 (1581354836636467200)]} 0 1
[junit4] 2> 1775163 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_real_time_get_shard1_replica_p4] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2]} 0 2
[junit4] 2> 1775164 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/|http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/
[junit4] 2> 1775164 INFO (qtp873174652-18327) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n2] webapp=/solr path=/get params={distrib=false&qt=/get&_stateVer_=pull_replica_test_real_time_get:5&omitHeader=true&shards.purpose=1&NOW=1508097492827&ids=2&isShard=true&shard.url=http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/|http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 1775165 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_p4] webapp=/solr path=/get params={qt=/get&_stateVer_=pull_replica_test_real_time_get:5&ids=2&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 1775166 INFO (qtp824552040-18334) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/get params={qt=/get&ids=2&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1775166 INFO (qtp873174652-18330) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n2] webapp=/solr path=/get params={qt=/get&ids=2&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1775168 INFO (qtp824552040-18338) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/|http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/
[junit4] 2> 1775169 INFO (qtp824552040-18339) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/get params={distrib=false&qt=/get&omitHeader=true&shards.purpose=1&NOW=1508097492832&ids=2&isShard=true&shard.url=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/|http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 1775170 INFO (qtp824552040-18338) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_p4] webapp=/solr path=/get params={qt=/get&ids=2&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 1775170 INFO (qtp873174652-18330) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/|http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/
[junit4] 2> 1775171 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n2] webapp=/solr path=/get params={distrib=false&qt=/get&omitHeader=true&shards.purpose=1&NOW=1508097492834&ids=2&isShard=true&shard.url=http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/|http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 1775171 INFO (qtp873174652-18330) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_p6] webapp=/solr path=/get params={qt=/get&ids=2&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 1775173 INFO (qtp824552040-18331) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/&wt=javabin&version=2}{add=[3 (1581354836649050112)]} 0 0
[junit4] 2> 1775173 INFO (qtp873174652-18332) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_real_time_get_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_p6/&wt=javabin&version=2}{add=[3 (1581354836649050112)]} 0 0
[junit4] 2> 1775178 INFO (qtp873174652-18328) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.u.p.LogUpdateProcessorFactory [pull_replica_test_real_time_get_shard1_replica_p6] webapp=/solr path=/update params={wt=javabin&version=2}{add=[3]} 0 6
[junit4] 2> 1775180 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/|http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/
[junit4] 2> 1775181 INFO (qtp824552040-18335) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/get params={distrib=false&qt=/get&_stateVer_=pull_replica_test_real_time_get:5&omitHeader=true&shards.purpose=1&NOW=1508097492844&ids=3&isShard=true&shard.url=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/|http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 1775181 INFO (qtp873174652-18326) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_p6] webapp=/solr path=/get params={qt=/get&_stateVer_=pull_replica_test_real_time_get:5&ids=3&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 1775182 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/get params={qt=/get&ids=3&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1775182 INFO (qtp873174652-18327) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n2] webapp=/solr path=/get params={qt=/get&ids=3&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1775183 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/|http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/
[junit4] 2> 1775184 INFO (qtp824552040-18334) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/get params={distrib=false&qt=/get&omitHeader=true&shards.purpose=1&NOW=1508097492847&ids=3&isShard=true&shard.url=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/|http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 1775184 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_p4] webapp=/solr path=/get params={qt=/get&ids=3&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 1775185 INFO (qtp873174652-18329) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/|http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/
[junit4] 2> 1775185 INFO (qtp824552040-18339) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/get params={distrib=false&qt=/get&omitHeader=true&shards.purpose=1&NOW=1508097492848&ids=3&isShard=true&shard.url=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/|http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 1775186 INFO (qtp873174652-18329) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_p6] webapp=/solr path=/get params={qt=/get&ids=3&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 1775187 INFO (qtp824552040-18339) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/get params={qt=/get&ids=0&ids=1&ids=2&ids=3&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1775188 INFO (qtp873174652-18323) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n2] webapp=/solr path=/get params={qt=/get&ids=0&ids=1&ids=2&ids=3&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1775188 INFO (qtp824552040-18331) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/|http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/
[junit4] 2> 1775189 INFO (qtp824552040-18335) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/get params={distrib=false&qt=/get&omitHeader=true&shards.purpose=1&NOW=1508097492852&ids=0,1,2,3&isShard=true&shard.url=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/|http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 1775190 INFO (qtp824552040-18331) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_p4] webapp=/solr path=/get params={qt=/get&ids=0&ids=1&ids=2&ids=3&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 1775191 INFO (qtp873174652-18330) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.h.c.RealTimeGetComponent LOOKUP_SLICE:shard1=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/|http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/
[junit4] 2> 1775191 INFO (qtp824552040-18337) [n:127.0.0.1:46484_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_n1] webapp=/solr path=/get params={distrib=false&qt=/get&omitHeader=true&shards.purpose=1&NOW=1508097492854&ids=0,1,2,3&isShard=true&shard.url=http://127.0.0.1:46484/solr/pull_replica_test_real_time_get_shard1_replica_n1/|http://127.0.0.1:53709/solr/pull_replica_test_real_time_get_shard1_replica_n2/&wt=javabin&version=2&shards.qt=/get} status=0 QTime=0
[junit4] 2> 1775192 INFO (qtp873174652-18330) [n:127.0.0.1:53709_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.S.Request [pull_replica_test_real_time_get_shard1_replica_p6] webapp=/solr path=/get params={qt=/get&ids=0&ids=1&ids=2&ids=3&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 1775192 INFO (TEST-TestPullReplica.testRealTimeGet-seed#[65B3A3DE4297EDBD]) [ ] o.a.s.c.TestPullReplica tearDown deleting collection
[junit4] 2> 1775193 INFO (qtp824552040-18334) [n:127.0.0.1:46484_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=pull_replica_test_real_time_get&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1775198 INFO (OverseerCollectionConfigSetProcessor-98834677005811718-127.0.0.1:46484_solr-n_0000000000) [n:127.0.0.1:46484_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1775198 INFO (OverseerThreadFactory-6060-thread-2-processing-n:127.0.0.1:46484_solr) [n:127.0.0.1:46484_solr ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
[junit4] 2> 1775199 INFO (qtp873174652-18332) [n:127.0.0.1:53709_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.pull_replica_test_real_time_get.shard1.replica_p6, tag=null
[junit4] 2> 1775199 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.pull_replica_test_real_time_get.shard1.replica_p4, tag=null
[junit4] 2> 1775199 INFO (qtp873174652-18332) [n:127.0.0.1:53709_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5cb1c4a: rootName = solr_53709, domain = solr.core.pull_replica_test_real_time_get.shard1.replica_p6, service url = null, agent id = null] for registry solr.core.pull_replica_test_real_time_get.shard1.replica_p6 / com.codahale.metrics.MetricRegistry@3c7cc273
[junit4] 2> 1775199 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@74b84a6c: rootName = solr_46484, domain = solr.core.pull_replica_test_real_time_get.shard1.replica_p4, service url = null, agent id = null] for registry solr.core.pull_replica_test_real_time_get.shard1.replica_p4 / com.codahale.metrics.MetricRegistry@5bf4644f
[junit4] 2> 1775215 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr ] o.a.s.c.ZkController pull_replica_test_real_time_get_shard1_replica_p4 stopping background replication from leader
[junit4] 2> 1775215 INFO (qtp824552040-18338) [n:127.0.0.1:46484_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.pull_replica_test_real_time_get.shard1.replica_n1, tag=null
[junit4] 2> 1775215 INFO (qtp824552040-18338) [n:127.0.0.1:46484_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5c5956ce: rootName = solr_46484, domain = solr.core.pull_replica_test_real_time_get.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.pull_replica_test_real_time_get.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@60efcebf
[junit4] 2> 1775215 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr ] o.a.s.c.SolrCore [pull_replica_test_real_time_get_shard1_replica_p4] CLOSING SolrCore org.apache.solr.core.SolrCore@5dafd257
[junit4] 2> 1775216 INFO (qtp873174652-18328) [n:127.0.0.1:53709_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.pull_replica_test_real_time_get.shard1.replica_n2, tag=null
[junit4] 2> 1775216 INFO (qtp873174652-18332) [n:127.0.0.1:53709_solr ] o.a.s.c.ZkController pull_replica_test_real_time_get_shard1_replica_p6 stopping background replication from leader
[junit4] 2> 1775216 INFO (qtp873174652-18328) [n:127.0.0.1:53709_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@124b80c6: rootName = solr_53709, domain = solr.core.pull_replica_test_real_time_get.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.pull_replica_test_real_time_get.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@60d7ff4
[junit4] 2> 1775217 INFO (qtp873174652-18332) [n:127.0.0.1:53709_solr ] o.a.s.c.SolrCore [pull_replica_test_real_time_get_shard1_replica_p6] CLOSING SolrCore org.apache.solr.core.SolrCore@3774afa5
[junit4] 2> 1775230 INFO (qtp824552040-18336) [n:127.0.0.1:46484_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for
[...truncated too long message...]
c935d4e: rootName = solr_46484, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@4a2cccab
[junit4] 2> 1919637 INFO (jetty-closer-2806-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 1919637 INFO (jetty-closer-2806-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@26e2248b: rootName = solr_53709, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@4a2cccab
[junit4] 2> 1919639 INFO (jetty-closer-2806-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 1919641 INFO (jetty-closer-2806-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 1919645 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f219d766c0010, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1919646 INFO (jetty-closer-2806-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6986dbf{/solr,null,UNAVAILABLE}
[junit4] 2> 1919647 INFO (zkCallback-2828-thread-2-processing-n:127.0.0.1:46484_solr) [n:127.0.0.1:46484_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 1919648 INFO (jetty-closer-2806-thread-1) [ ] o.a.s.c.Overseer Overseer (id=98834677005811723-127.0.0.1:46484_solr-n_0000000002) closing
[junit4] 2> 1919650 INFO (OverseerStateUpdate-98834677005811723-127.0.0.1:46484_solr-n_0000000002) [n:127.0.0.1:46484_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:46484_solr
[junit4] 2> 1919653 WARN (OverseerAutoScalingTriggerThread-98834677005811723-127.0.0.1:46484_solr-n_0000000002) [n:127.0.0.1:46484_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 1919660 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f219d766c000b, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1919661 INFO (jetty-closer-2806-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@345a0a0{/solr,null,UNAVAILABLE}
[junit4] 2> 1919663 ERROR (SUITE-TestPullReplica-seed#[65B3A3DE4297EDBD]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1919663 INFO (SUITE-TestPullReplica-seed#[65B3A3DE4297EDBD]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:47484 47484
[junit4] 2> 1926223 INFO (Thread-5455) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:47484 47484
[junit4] 2> 1926224 WARN (Thread-5455) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 19 /solr/configs/conf
[junit4] 2> 7 /solr/aliases.json
[junit4] 2> 4 /solr/security.json
[junit4] 2> 3 /solr/clusterprops.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 22 /solr/collections/pull_replica_test_create_delete/state.json
[junit4] 2> 21 /solr/collections/pull_replica_test_remove_all_writer_replicas/state.json
[junit4] 2> 20 /solr/collections/pull_replica_test_kill_pull_replica/state.json
[junit4] 2> 19 /solr/collections/pull_replica_test_add_remove_pull_replica/state.json
[junit4] 2> 16 /solr/collections/pull_replica_test_kill_leader/state.json
[junit4] 2> 13 /solr/collections/pull_replica_test_real_time_get/state.json
[junit4] 2> 13 /solr/collections/pull_replica_test_add_docs/state.json
[junit4] 2> 7 /solr/clusterprops.json
[junit4] 2> 7 /solr/clusterstate.json
[junit4] 2> 3 /solr/autoscaling.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/collections
[junit4] 2> 5 /solr/live_nodes
[junit4] 2> 3 /solr/overseer/queue
[junit4] 2> 3 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 3 /solr/overseer/collection-queue-work
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_65B3A3DE4297EDBD-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {foo=PostingsFormat(name=LuceneVarGapDocFreqInterval), foo_s=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=Lucene50(blocksize=128)}, docValues:{_version_=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1522, maxMBSortInHeap=5.884976479259683, sim=RandomSimilarity(queryNorm=false): {}, locale=fr-LU, timezone=Australia/Broken_Hill
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=264589768,total=518979584
[junit4] 2> NOTE: All tests run in this JVM: [TestXmlQParserPlugin, TestTolerantUpdateProcessorCloud, HdfsBasicDistributedZk2Test, FieldMutatingUpdateProcessorTest, CoreAdminRequestStatusTest, BlockJoinFacetRandomTest, StressHdfsTest, DocValuesMissingTest, SolrPluginUtilsTest, TestJsonFacets, TermsComponentTest, TestMergePolicyConfig, TestSolrQueryParser, LoggingHandlerTest, ClassificationUpdateProcessorFactoryTest, TestCloudInspectUtil, SuggesterFSTTest, TestRecoveryHdfs, TestLuceneIndexBackCompat, TestConfigReload, PropertiesRequestHandlerTest, LeaderInitiatedRecoveryOnShardRestartTest, StatsComponentTest, TestSQLHandlerNonCloud, TestExportWriter, SearchHandlerTest, DOMUtilTest, DistributedMLTComponentTest, TestPolicyCloud, TestDocSet, TestSolrJ, TestSolrDeletionPolicy1, DirectSolrSpellCheckerTest, DistributedFacetPivotSmallTest, DeleteInactiveReplicaTest, RestartWhileUpdatingTest, FileBasedSpellCheckerTest, BasicFunctionalityTest, AutoscalingHistoryHandlerTest, TestImpersonationWithHadoopAuth, StatelessScriptUpdateProcessorFactoryTest, DistributedIntervalFacetingTest, TestNestedDocsSort, ConvertedLegacyTest, HLLSerializationTest, TestLeaderInitiatedRecoveryThread, TestLuceneMatchVersion, TestManagedSynonymGraphFilterFactory, TestComponentsName, HdfsDirectoryFactoryTest, TestStressCloudBlindAtomicUpdates, TestRandomFaceting, HdfsRestartWhileUpdatingTest, TestRecovery, RecoveryAfterSoftCommitTest, AtomicUpdateProcessorFactoryTest, CopyFieldTest, TestFieldTypeCollectionResource, BufferStoreTest, ConfigureRecoveryStrategyTest, ConfigSetsAPITest, HdfsUnloadDistributedZkTest, TestBackupRepositoryFactory, TestDistributedMissingSort, TestAddFieldRealTimeGet, TestSchemaNameResource, TestClusterProperties, CloudMLTQParserTest, SystemInfoHandlerTest, CollectionsAPIDistributedZkTest, PluginInfoTest, ChaosMonkeySafeLeaderTest, UpdateRequestProcessorFactoryTest, TestPKIAuthenticationPlugin, TestFastOutputStream, BJQParserTest, SolrIndexConfigTest, TestOnReconnectListenerSupport, TestConfig, TestRTimerTree, BigEndianAscendingWordSerializerTest, SuggestComponentContextFilterQueryTest, TestFieldCollectionResource, CdcrReplicationHandlerTest, SpellCheckCollatorTest, TestFastLRUCache, SpellPossibilityIteratorTest, InfoHandlerTest, TestCrossCoreJoin, AnalyticsQueryTest, AddBlockUpdateTest, TestSSLRandomization, PeerSyncWithIndexFingerprintCachingTest, CdcrVersionReplicationTest, TestFieldCacheVsDocValues, DocumentBuilderTest, TestReloadAndDeleteDocs, BadComponentTest, TestStressLiveNodes, TestHttpShardHandlerFactory, TestValueSourceCache, ExternalFileFieldSortTest, TestBlendedInfixSuggestions, CollectionsAPISolrJTest, OpenCloseCoreStressTest, TestFieldCacheSort, TestPostingsSolrHighlighter, TestTestInjection, BasicDistributedZk2Test, TestAuthenticationFramework, TestSolrXml, TestSearchPerf, TestWordDelimiterFilterFactory, FullSolrCloudDistribCmdsTest, SignatureUpdateProcessorFactoryTest, ZkStateReaderTest, CollectionTooManyReplicasTest, PKIAuthenticationIntegrationTest, ShardSplitTest, SpellCheckComponentTest, BlockCacheTest, SolrGraphiteReporterTest, TestRTGBase, TestJoin, TestConfigsApi, TestNumericRangeQuery64, BooleanFieldTest, TestReplicationHandlerBackup, TestSolrConfigHandlerConcurrent, MBeansHandlerTest, ResponseHeaderTest, TestHalfAndHalfDocValues, PreAnalyzedFieldTest, SimpleCollectionCreateDeleteTest, TestElisionMultitermQuery, ShardRoutingCustomTest, TestSolr4Spatial, ShowFileRequestHandlerTest, CacheHeaderTest, TestIndexSearcher, PolyFieldTest, ExecutePlanActionTest, ImplicitSnitchTest, OverseerCollectionConfigSetProcessorTest, TermVectorComponentDistributedTest, TestComplexPhraseLeadingWildcard, DocumentAnalysisRequestHandlerTest, TestPayloadCheckQParserPlugin, TestInPlaceUpdatesStandalone, SparseHLLTest, TestQueryUtils, TestMacroExpander, DistributedQueryElevationComponentTest, TestNumericTerms64, DistributedFacetExistsSmallTest, TestSchemalessBufferedUpdates, HdfsRecoverLeaseTest, TestHashQParserPlugin, TestPullReplica]
[junit4] Completed [345/744 (1!)] on J1 in 155.41s, 10 tests, 1 failure, 1 skipped <<< FAILURES!
[...truncated 48336 lines...]
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 245 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/245/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC
2 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
Error Message:
3 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 1) Thread[id=33608, name=jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 2) Thread[id=33609, name=jetty-launcher-5069-thread-1-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 3) Thread[id=33548, name=jetty-launcher-5069-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323) at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105) at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41) at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244) at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44) at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61) at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
1) Thread[id=33608, name=jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
2) Thread[id=33609, name=jetty-launcher-5069-thread-1-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
3) Thread[id=33548, name=jetty-launcher-5069-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
at __randomizedtesting.SeedInfo.seed([6FFD94895A07389C]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=33608, name=jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=33608, name=jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
at __randomizedtesting.SeedInfo.seed([6FFD94895A07389C]:0)
Build Log:
[...truncated 13371 lines...]
[junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_6FFD94895A07389C-001/init-core-data-001
[junit4] 2> 3439856 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[6FFD94895A07389C]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 3439858 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[6FFD94895A07389C]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 3439858 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[6FFD94895A07389C]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 3439967 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[6FFD94895A07389C]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_6FFD94895A07389C-001/tempDir-001
[junit4] 2> 3439968 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[6FFD94895A07389C]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3439968 INFO (Thread-10182) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3439968 INFO (Thread-10182) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3439971 ERROR (Thread-10182) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3440069 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[6FFD94895A07389C]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:35405
[junit4] 2> 3440093 INFO (jetty-launcher-5069-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3440094 INFO (jetty-launcher-5069-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3440097 INFO (jetty-launcher-5069-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@40469849{/solr,null,AVAILABLE}
[junit4] 2> 3440099 INFO (jetty-launcher-5069-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7d55ac02{HTTP/1.1,[http/1.1]}{127.0.0.1:45060}
[junit4] 2> 3440099 INFO (jetty-launcher-5069-thread-1) [ ] o.e.j.s.Server Started @3442937ms
[junit4] 2> 3440099 INFO (jetty-launcher-5069-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45060}
[junit4] 2> 3440099 ERROR (jetty-launcher-5069-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3440099 INFO (jetty-launcher-5069-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@c18d644{/solr,null,AVAILABLE}
[junit4] 2> 3440099 INFO (jetty-launcher-5069-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 3440100 INFO (jetty-launcher-5069-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3440100 INFO (jetty-launcher-5069-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 3440100 INFO (jetty-launcher-5069-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-16T13:48:49.387Z
[junit4] 2> 3440100 INFO (jetty-launcher-5069-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@39cd2a02{HTTP/1.1,[http/1.1]}{127.0.0.1:53257}
[junit4] 2> 3440100 INFO (jetty-launcher-5069-thread-2) [ ] o.e.j.s.Server Started @3442938ms
[junit4] 2> 3440100 INFO (jetty-launcher-5069-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=53257}
[junit4] 2> 3440101 ERROR (jetty-launcher-5069-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3440101 INFO (jetty-launcher-5069-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 3440101 INFO (jetty-launcher-5069-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3440101 INFO (jetty-launcher-5069-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 3440101 INFO (jetty-launcher-5069-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-16T13:48:49.388Z
[junit4] 2> 3440105 INFO (jetty-launcher-5069-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3440106 INFO (jetty-launcher-5069-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3440118 INFO (jetty-launcher-5069-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35405/solr
[junit4] 2> 3440118 INFO (jetty-launcher-5069-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35405/solr
[junit4] 2> 3440273 INFO (jetty-launcher-5069-thread-2) [n:127.0.0.1:53257_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3440274 INFO (jetty-launcher-5069-thread-1) [n:127.0.0.1:45060_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3440275 INFO (jetty-launcher-5069-thread-2) [n:127.0.0.1:53257_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:53257_solr
[junit4] 2> 3440276 INFO (jetty-launcher-5069-thread-1) [n:127.0.0.1:45060_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45060_solr
[junit4] 2> 3440276 INFO (jetty-launcher-5069-thread-2) [n:127.0.0.1:53257_solr ] o.a.s.c.Overseer Overseer (id=98838887088521222-127.0.0.1:53257_solr-n_0000000000) starting
[junit4] 2> 3440279 INFO (zkCallback-5081-thread-1-processing-n:127.0.0.1:53257_solr) [n:127.0.0.1:53257_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3440282 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45060_solr) [n:127.0.0.1:45060_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3440292 INFO (jetty-launcher-5069-thread-2) [n:127.0.0.1:53257_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53257_solr
[junit4] 2> 3440294 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45060_solr) [n:127.0.0.1:45060_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3440298 INFO (zkCallback-5081-thread-1-processing-n:127.0.0.1:53257_solr) [n:127.0.0.1:53257_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3440314 INFO (jetty-launcher-5069-thread-1) [n:127.0.0.1:45060_solr ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 3440315 WARN (jetty-launcher-5069-thread-1) [n:127.0.0.1:45060_solr ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
[junit4] 2> 3440315 INFO (jetty-launcher-5069-thread-1) [n:127.0.0.1:45060_solr ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
[junit4] 2> 3440316 INFO (jetty-launcher-5069-thread-1) [n:127.0.0.1:45060_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 3440318 INFO (jetty-launcher-5069-thread-1-EventThread) [n:127.0.0.1:45060_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 3440337 INFO (jetty-launcher-5069-thread-2) [n:127.0.0.1:53257_solr ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 3440337 WARN (jetty-launcher-5069-thread-2) [n:127.0.0.1:53257_solr ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
[junit4] 2> 3440337 INFO (jetty-launcher-5069-thread-2) [n:127.0.0.1:53257_solr ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
[junit4] 2> 3440342 INFO (jetty-launcher-5069-thread-2) [n:127.0.0.1:53257_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 3440343 WARN (jetty-launcher-5069-thread-1) [n:127.0.0.1:45060_solr ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
[junit4] 2> 3440344 INFO (jetty-launcher-5069-thread-2-EventThread) [n:127.0.0.1:53257_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 3440373 WARN (jetty-launcher-5069-thread-2) [n:127.0.0.1:53257_solr ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
[junit4] 2> 3440416 INFO (jetty-launcher-5069-thread-1) [n:127.0.0.1:45060_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45060.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c957992
[junit4] 2> 3440416 INFO (jetty-launcher-5069-thread-2) [n:127.0.0.1:53257_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53257.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c957992
[junit4] 2> 3440422 INFO (jetty-launcher-5069-thread-1) [n:127.0.0.1:45060_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45060.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c957992
[junit4] 2> 3440422 INFO (jetty-launcher-5069-thread-2) [n:127.0.0.1:53257_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53257.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c957992
[junit4] 2> 3440422 INFO (jetty-launcher-5069-thread-1) [n:127.0.0.1:45060_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45060.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c957992
[junit4] 2> 3440422 INFO (jetty-launcher-5069-thread-2) [n:127.0.0.1:53257_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53257.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c957992
[junit4] 2> 3440423 INFO (jetty-launcher-5069-thread-1) [n:127.0.0.1:45060_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_6FFD94895A07389C-001/tempDir-001/node1/.
[junit4] 2> 3440423 INFO (jetty-launcher-5069-thread-2) [n:127.0.0.1:53257_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_6FFD94895A07389C-001/tempDir-001/node2/.
[junit4] 2> 3440609 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyWrongHost
[junit4] 2> 3446980 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyWrongHost
[junit4] 2> 3447137 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateAnyHostAnyUser
[junit4] 2> 3447151 WARN (qtp622402794-33493) [ ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 3447155 INFO (qtp622402794-33493) [n:127.0.0.1:45060_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3447155 INFO (qtp622402794-33493) [n:127.0.0.1:45060_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3447158 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateAnyHostAnyUser
[junit4] 2> 3447438 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigGroups
[junit4] 2> 3447439 WARN (qtp622402794-33494) [ ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 3447446 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigGroups
[junit4] 2> 3447646 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigHosts
[junit4] 2> 3447647 WARN (qtp622402794-33495) [ ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 3447649 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigHosts
[junit4] 2> 3447830 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidRemote
[junit4] 2> 3447831 WARN (qtp622402794-33496) [ ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 3447839 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidRemote
[junit4] 2> 3448036 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testForwarding
[junit4] 2> 3449192 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f2571b2e8000a, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3449197 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[6FFD94895A07389C]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3449199 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[6FFD94895A07389C]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35405/solr ready
[junit4] 2> 3449201 INFO (qtp622402794-33490) [n:127.0.0.1:45060_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3449207 INFO (OverseerThreadFactory-12292-thread-1-processing-n:127.0.0.1:53257_solr) [n:127.0.0.1:53257_solr ] o.a.s.c.CreateCollectionCmd Create collection forwardingCollection
[junit4] 2> 3449322 INFO (OverseerStateUpdate-98838887088521222-127.0.0.1:53257_solr-n_0000000000) [n:127.0.0.1:53257_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"forwardingCollection",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"forwardingCollection_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:45060/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 3449524 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica_n1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3449525 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 3449630 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45060_solr) [n:127.0.0.1:45060_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3449630 INFO (zkCallback-5080-thread-2-processing-n:127.0.0.1:45060_solr) [n:127.0.0.1:45060_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3450559 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 3450585 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.IndexSchema [forwardingCollection_shard1_replica_n1] Schema name=test
[junit4] 2> 3450667 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3450685 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'forwardingCollection_shard1_replica_n1' using configuration from collection forwardingCollection, trusted=true
[junit4] 2> 3450686 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45060.solr.core.forwardingCollection.shard1.replica_n1' (registry 'solr.core.forwardingCollection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c957992
[junit4] 2> 3450686 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3450686 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [[forwardingCollection_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_6FFD94895A07389C-001/tempDir-001/node1/forwardingCollection_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_6FFD94895A07389C-001/tempDir-001/node1/./forwardingCollection_shard1_replica_n1/data/]
[junit4] 2> 3450688 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 3450738 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3450738 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3450740 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3450740 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3450741 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=83.9169921875, floorSegmentMB=1.18359375, forceMergeDeletesPctAllowed=3.186662519454276, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 3450741 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@41eea24[forwardingCollection_shard1_replica_n1] main]
[junit4] 2> 3450743 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 3450762 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 3450789 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3450790 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581422204762980352
[junit4] 2> 3450790 INFO (searcherExecutor-12297-thread-1-processing-n:127.0.0.1:45060_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 3450795 INFO (searcherExecutor-12297-thread-1-processing-n:127.0.0.1:45060_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 3450795 INFO (searcherExecutor-12297-thread-1-processing-n:127.0.0.1:45060_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 3450795 INFO (searcherExecutor-12297-thread-1-processing-n:127.0.0.1:45060_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 3450795 INFO (searcherExecutor-12297-thread-1-processing-n:127.0.0.1:45060_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 3450796 INFO (searcherExecutor-12297-thread-1-processing-n:127.0.0.1:45060_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 3450796 INFO (searcherExecutor-12297-thread-1-processing-n:127.0.0.1:45060_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 3450797 INFO (searcherExecutor-12297-thread-1-processing-n:127.0.0.1:45060_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 3450797 INFO (searcherExecutor-12297-thread-1-processing-n:127.0.0.1:45060_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 3450799 INFO (searcherExecutor-12297-thread-1-processing-n:127.0.0.1:45060_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica_n1] Registered new searcher Searcher@41eea24[forwardingCollection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3450799 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3450799 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3450800 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45060/solr/forwardingCollection_shard1_replica_n1/
[junit4] 2> 3450800 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3450800 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:45060/solr/forwardingCollection_shard1_replica_n1/ has no replicas
[junit4] 2> 3450800 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3450805 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45060/solr/forwardingCollection_shard1_replica_n1/ shard1
[junit4] 2> 3450908 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45060_solr) [n:127.0.0.1:45060_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3450908 INFO (zkCallback-5080-thread-2-processing-n:127.0.0.1:45060_solr) [n:127.0.0.1:45060_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3450913 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3450916 INFO (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica_n1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1392
[junit4] 2> 3450925 INFO (qtp622402794-33490) [n:127.0.0.1:45060_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3451020 INFO (zkCallback-5080-thread-2-processing-n:127.0.0.1:45060_solr) [n:127.0.0.1:45060_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3451020 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45060_solr) [n:127.0.0.1:45060_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3451214 INFO (OverseerCollectionConfigSetProcessor-98838887088521222-127.0.0.1:53257_solr-n_0000000000) [n:127.0.0.1:53257_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3451928 INFO (qtp622402794-33490) [n:127.0.0.1:45060_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2} status=0 QTime=2727
[junit4] 2> 3451929 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[6FFD94895A07389C]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: forwardingCollection failOnTimeout:true timeout (sec):100
[junit4] 2> 3451930 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[6FFD94895A07389C]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: forwardingCollection
[junit4] 2> 3451935 INFO (qtp622402794-33495) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.S.Request [forwardingCollection_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 3451950 INFO (qtp737432537-33500) [n:127.0.0.1:53257_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 3451954 INFO (qtp622402794-33496) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.S.Request [forwardingCollection_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 3451956 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testForwarding
[junit4] 2> 3452138 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidProxyUser
[junit4] 2> 3452140 WARN (qtp622402794-33492) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 3452142 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidProxyUser
[junit4] 2> 3452314 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidGroup
[junit4] 2> 3452314 WARN (qtp622402794-33490) [n:127.0.0.1:45060_solr ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 3452653 WARN (qtp622402794-33490) [n:127.0.0.1:45060_solr ] o.a.h.s.ShellBasedUnixGroupsMapping got exception trying to get groups for user bar: id: invalid user name: "bar"
[junit4] 2> id: invalid user name: "bar"
[junit4] 2>
[junit4] 2> 3452654 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidGroup
[junit4] 2> 3452841 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateGroup
[junit4] 2> 3452842 WARN (qtp622402794-33494) [ ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 3452842 INFO (qtp622402794-33494) [n:127.0.0.1:45060_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=jenkins&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3452842 INFO (qtp622402794-33494) [n:127.0.0.1:45060_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=jenkins&action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3452843 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateGroup
[junit4] 2> 3453016 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNullProxyUser
[junit4] 2> 3453016 WARN (qtp622402794-33495) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 3453017 INFO (qtp622402794-33495) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.SolrDispatchFilter Error authenticating
[junit4] 2> java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
[junit4] 2> at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
[junit4] 2> at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
[junit4] 2> at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
[junit4] 2> at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:134)
[junit4] 2> at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
[junit4] 2> at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3453017 WARN (qtp622402794-33495) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.e.j.s.ServletHandler /solr/admin/collections
[junit4] 2> org.apache.solr.common.SolrException: Error during request authentication,
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:457)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
[junit4] 2> at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
[junit4] 2> at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
[junit4] 2> at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
[junit4] 2> at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:134)
[junit4] 2> at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
[junit4] 2> at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
[junit4] 2> ... 26 more
[junit4] 2> 3453018 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNullProxyUser
[junit4] 2> 3453197 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateHost
[junit4] 2> 3453198 WARN (qtp622402794-33497) [ ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 3453199 INFO (qtp622402794-33497) [n:127.0.0.1:45060_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3453199 INFO (qtp622402794-33497) [n:127.0.0.1:45060_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3453199 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateHost
[junit4] 2> 3453369 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyUnknownRemote
[junit4] 2> 3453369 WARN (qtp622402794-33496) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 3469208 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 18186ms for sessionid 0x15f2571b2e80005
[junit4] 2> 3469208 WARN (jetty-launcher-5069-thread-2-SendThread(127.0.0.1:35405)) [n:127.0.0.1:53257_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15986ms for sessionid 0x15f2571b2e80006
[junit4] 2> 3469208 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[6FFD94895A07389C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyUnknownRemote
[junit4] 2> 3469208 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f2571b2e80005, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3469210 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f2571b2e80006, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3469311 WARN (zkCallback-5080-thread-1-processing-n:127.0.0.1:45060_solr) [n:127.0.0.1:45060_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@517da89d name: ZooKeeperConnection Watcher:127.0.0.1:35405/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 3469311 WARN (zkCallback-5080-thread-1-processing-n:127.0.0.1:45060_solr) [n:127.0.0.1:45060_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 3469312 WARN (zkCallback-5081-thread-3-processing-n:127.0.0.1:53257_solr) [n:127.0.0.1:53257_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2b235e17 name: ZooKeeperConnection Watcher:127.0.0.1:35405/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 3469312 WARN (zkCallback-5081-thread-3-processing-n:127.0.0.1:53257_solr) [n:127.0.0.1:53257_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 3469313 INFO (jetty-launcher-5069-thread-1-EventThread) [n:127.0.0.1:45060_solr ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
[junit4] 2> 3469313 INFO (jetty-closer-5070-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@7d55ac02{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 3469317 INFO (jetty-closer-5070-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1854184139
[junit4] 2> 3469318 INFO (jetty-closer-5070-thread-2) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@39cd2a02{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 3469320 INFO (jetty-closer-5070-thread-2) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=417720348
[junit4] 2> 3469321 INFO (jetty-closer-5070-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 3469321 INFO (jetty-closer-5070-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@67e51080: rootName = solr_45060, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@68fbc35f
[junit4] 2> 3469323 INFO (jetty-closer-5070-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 3469323 INFO (jetty-closer-5070-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@c99fbed: rootName = solr_53257, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@49f9218c
[junit4] 2> 3469329 INFO (jetty-closer-5070-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 3469329 INFO (jetty-closer-5070-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@69392977: rootName = solr_53257, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@5710c0d7
[junit4] 2> 3469331 INFO (jetty-closer-5070-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 3469331 INFO (jetty-closer-5070-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@44918bcf: rootName = solr_45060, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@5710c0d7
[junit4] 2> 3469333 INFO (jetty-closer-5070-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 3469333 INFO (jetty-closer-5070-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2761bbcb: rootName = solr_53257, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@532f5eed
[junit4] 2> 3469334 INFO (jetty-closer-5070-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 3469334 INFO (jetty-closer-5070-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@207a9534: rootName = solr_45060, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@532f5eed
[junit4] 2> 3470404 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15f2571b2e80007 has expired
[junit4] 2> 3471082 INFO (jetty-closer-5070-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 3471082 INFO (jetty-closer-5070-thread-2) [ ] o.a.s.c.Overseer Overseer (id=98838887088521222-127.0.0.1:53257_solr-n_0000000000) closing
[junit4] 2> 3471083 WARN (OverseerAutoScalingTriggerThread-98838887088521222-127.0.0.1:53257_solr-n_0000000000) [n:127.0.0.1:53257_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 3471088 INFO (OverseerStateUpdate-98838887088521222-127.0.0.1:53257_solr-n_0000000000) [n:127.0.0.1:53257_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:53257_solr
[junit4] 2> 3471092 ERROR (Thread[Thread-10201,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:53257_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 3471180 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45060_solr) [n:127.0.0.1:45060_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45060_solr
[junit4] 2> 3471180 INFO (zkCallback-5080-thread-5-processing-n:127.0.0.1:45060_solr) [n:127.0.0.1:45060_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 3471182 INFO (jetty-closer-5070-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 3471182 INFO (coreCloseExecutor-12303-thread-1) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@c1c4df4
[junit4] 2> 3471182 INFO (coreCloseExecutor-12303-thread-1) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.forwardingCollection.shard1.replica_n1, tag=203181556
[junit4] 2> 3471183 INFO (coreCloseExecutor-12303-thread-1) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5685572d: rootName = solr_45060, domain = solr.core.forwardingCollection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.forwardingCollection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@3b7b8a86
[junit4] 2> 3471203 INFO (coreCloseExecutor-12303-thread-1) [n:127.0.0.1:45060_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.forwardingCollection.shard1.leader, tag=203181556
[junit4] 2> 3471209 ERROR (Thread[Thread-10199,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:45060_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 3471216 INFO (jetty-closer-5070-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@40469849{/solr,null,UNAVAILABLE}
[junit4] 2> 3471261 WARN (jetty-launcher-5069-thread-2-SendThread(127.0.0.1:35405)) [n:127.0.0.1:53257_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15f2571b2e80008 has expired
[junit4] 2> 3471365 INFO (jetty-closer-5070-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@c18d644{/solr,null,UNAVAILABLE}
[junit4] 2> 3471368 ERROR (SUITE-TestSolrCloudWithSecureImpersonation-seed#[6FFD94895A07389C]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3471369 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[6FFD94895A07389C]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35405 35405
[junit4] 2> 3482197 INFO (Thread-10182) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35405 35405
[junit4] 2> 3482206 WARN (Thread-10182) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/aliases.json
[junit4] 2> 3 /solr/clusterprops.json
[junit4] 2> 2 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/collections/forwardingCollection/state.json
[junit4] 2> 3 /solr/clusterstate.json
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/live_nodes
[junit4] 2> 3 /solr/collections
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
[junit4] 2>
[junit4] 2> 3482645 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[6FFD94895A07389C]-worker) [ ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
[junit4] 2> Oct 16, 2017 1:49:31 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
[junit4] 2> 3499380 ERROR (jetty-launcher-5069-thread-1-EventThread) [n:127.0.0.1:45060_solr ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
[junit4] 2> java.lang.IllegalStateException: Client is not started
[junit4] 2> at com.google.common.base.Preconditions.checkState(Preconditions.java:149)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:113)
[junit4] 2> at org.apache.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:477)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceFacade.getZooKeeper(NamespaceFacade.java:112)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl$4.call(GetDataBuilderImpl.java:302)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl$4.call(GetDataBuilderImpl.java:291)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:107)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> 3499388 WARN (jetty-launcher-5069-thread-1-EventThread) [n:127.0.0.1:45060_solr ] o.a.c.ConnectionState Session expired event received
[junit4] 2> 3499391 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
[junit4] 2> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
[junit4] 2> 3500493 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
[junit4] 2> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
[junit4] 2> 3501595 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
[junit4] 2> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
[junit4] 2> 3502697 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
[junit4] 2> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
[junit4] 2> Oct 16, 2017 1:49:52 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> SEVERE: 3 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
[junit4] 2> 1) Thread[id=33608, name=jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] 2> 2) Thread[id=33609, name=jetty-launcher-5069-thread-1-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> at sun.misc.Unsafe.park(Native Method)
[junit4] 2> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] 2> at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
[junit4] 2> 3) Thread[id=33548, name=jetty-launcher-5069-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> at sun.misc.Unsafe.park(Native Method)
[junit4] 2> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> Oct 16, 2017 1:49:52 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: Starting to interrupt leaked threads:
[junit4] 2> 1) Thread[id=33608, name=jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> 2) Thread[id=33609, name=jetty-launcher-5069-thread-1-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> 3) Thread[id=33548, name=jetty-launcher-5069-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> 3502958 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.c.StaticHostProvider Unexpected exception
[junit4] 2> java.lang.InterruptedException: sleep interrupted
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] 2> 3502958 ERROR (jetty-launcher-5069-thread-2-EventThread) [n:127.0.0.1:53257_solr ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
[junit4] 2> java.lang.InterruptedException
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> 3502959 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
[junit4] 2> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
[junit4] 2> 3502958 ERROR (jetty-launcher-5069-thread-1-EventThread) [n:127.0.0.1:45060_solr ] o.a.z.ClientCnxn Event thread exiting due to interruption
[junit4] 2> java.lang.InterruptedException
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048)
[junit4] 2> at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
[junit4] 2> 3503459 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.c.StaticHostProvider Unexpected exception
[junit4] 2> java.lang.InterruptedException: sleep interrupted
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] 2> 3503460 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
[junit4] 2> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
[junit4] 2> 3503960 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.c.StaticHostProvider Unexpected exception
[junit4] 2> java.lang.InterruptedException: sleep interrupted
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] 2> 3503961 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
[junit4] 2> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
[junit4] 2> 3504460 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.c.StaticHostProvider Unexpected exception
[junit4] 2> java.lang.InterruptedException: sleep interrupted
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] 2> 3504461 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
[junit4] 2> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
[junit4] 2> 3504961 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.c.StaticHostProvider Unexpected exception
[junit4] 2> java.lang.InterruptedException: sleep interrupted
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] 2> 3504961 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
[junit4] 2> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
[junit4] 2> 3505463 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.c.StaticHostProvider Unexpected exception
[junit4] 2> java.lang.InterruptedException: sleep interrupted
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] 2> 3505463 WARN (jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405)) [n:127.0.0.1:45060_solr ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
[junit4] 2> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
[junit4] 2> Oct 16, 2017 1:49:55 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> SEVERE: There are still zombie threads that couldn't be terminated:
[junit4] 2> 1) Thread[id=33608, name=jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] 2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=20266, maxDocsPerChunk=4, blockSize=528), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=20266, blockSize=528)), sim=RandomSimilarity(queryNorm=true): {}, locale=und, timezone=America/North_Dakota/Beulah
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=2,free=84364856,total=536870912
[junit4] 2> NOTE: All tests run in this JVM: [SuggestComponentTest, SolrInfoBeanTest, MultiTermTest, TestCloudNestedDocsSort, TestLegacyFieldReuse, SolrCLIZkUtilsTest, TestFileDictionaryLookup, ReplicationFactorTest, TestEmbeddedSolrServerSchemaAPI, TestDynamicLoading, TestDownShardTolerantSearch, PrimitiveFieldTypeTest, ClusterStateUpdateTest, SolrXmlInZkTest, HdfsSyncSliceTest, RollingRestartTest, MetricUtilsTest, MetricsHandlerTest, TestJsonRequest, TestDFISimilarityFactory, ChaosMonkeyNothingIsSafeWithPullReplicasTest, TestNRTOpen, SpatialRPTFieldTypeTest, IndexSchemaTest, TestMinMaxOnMultiValuedField, SolrIndexSplitterTest, TestSubQueryTransformerCrossCore, TestFaceting, JavabinLoaderTest, NodeMutatorTest, Te
[...truncated too long message...]
yErrorTest, TestUninvertingReader, NotRequiredUniqueKeyTest, NumericFieldsTest, AddSchemaFieldsUpdateProcessorFactoryTest, TriggerIntegrationTest, TestSimpleQParserPlugin, AlternateDirectoryTest, TestUniqueKeyFieldResource, TestUnifiedSolrHighlighter, QueryParsingTest, TestDistributedSearch, UpdateLogTest, TestSweetSpotSimilarityFactory, TestSolrConfigHandler, SortByFunctionTest, TestReloadDeadlock, DistributedFacetPivotWhiteBoxTest, TestSolr4Spatial2, RegexBoostProcessorTest, TestCloudDeleteByQuery, SharedFSAutoReplicaFailoverTest, TestLMDirichletSimilarityFactory, HdfsRecoveryZkTest, LeaderFailoverAfterPartitionTest, TestCopyFieldCollectionResource, TestFieldTypeResource, TestShardHandlerFactory, SolrCoreMetricManagerTest, TestPrepRecovery, UUIDFieldTest, DeleteReplicaTest, OutputWriterTest, TestWriterPerf, UniqFieldsUpdateProcessorFactoryTest, TestXIncludeConfig, TestDynamicFieldResource, CheckHdfsIndexTest, HdfsChaosMonkeySafeLeaderTest, TestSchemaResource, TemplateUpdateProcessorTest, HdfsWriteToMultipleCollectionsTest, HdfsNNFailoverTest, TestLazyCores, TestCoreContainer, TestTolerantUpdateProcessorRandomCloud, ConjunctionSolrSpellCheckerTest, TestLegacyTerms, LeaderElectionTest, EnumFieldTest, ZkFailoverTest, TriLevelCompositeIdRoutingTest, BlobRepositoryCloudTest, TestRequestStatusCollectionAPI, TestRetrieveFieldsOptimizer, TestNumericRangeQuery32, TestDelegationWithHadoopAuth, TestHighlightDedupGrouping, ZkCLITest, ConcurrentDeleteAndCreateCollectionTest, BigEndianAscendingWordDeserializerTest, TimeZoneUtilsTest, SmileWriterTest, TestManagedSchemaThreadSafety, TestStressUserVersions, SolrIndexMetricsTest, TestMiniSolrCloudClusterSSL, SuggesterWFSTTest, TolerantUpdateProcessorTest, TestCoreAdminApis, TestFieldSortValues, TestManagedSynonymFilterFactory, TestInitQParser, TestSolrCoreProperties, TestExactSharedStatsCache, SimplePostToolTest, TestAtomicUpdateErrorCases, TestScoreJoinQPNoScore, TestLocalFSCloudBackupRestore, CollectionStateFormat2Test, SpatialFilterTest, BlockJoinFacetSimpleTest, TestSolrFieldCacheBean, TestLRUStatsCache, CoreMergeIndexesAdminHandlerTest, PreAnalyzedUpdateProcessorTest, TestObjectReleaseTracker, SecurityConfHandlerTest, SolrShardReporterTest, TestIBSimilarityFactory, HdfsTlogReplayBufferedWhileIndexingTest, TestPHPSerializedResponseWriter, ShardRoutingTest, TestStreamBody, CloneFieldUpdateProcessorFactoryTest, DefaultValueUpdateProcessorTest, QueryElevationComponentTest, TestManagedSchema, TestEmbeddedSolrServerAdminHandler, ReturnFieldsTest, TestMultiWordSynonyms, CachingDirectoryFactoryTest, CSVRequestHandlerTest, AnalysisAfterCoreReloadTest, AssignTest, TestSurroundQueryParser, RecoveryZkTest, CdcrRequestHandlerTest, TestPerFieldSimilarityWithDefaultOverride, TestFieldCacheSortRandom, UnloadDistributedZkTest, DistribDocExpirationUpdateProcessorTest, SpatialHeatmapFacetsTest, BlockDirectoryTest, TestConfigOverlay, TestFuzzyAnalyzedSuggestions, TestDefaultStatsCache, TestFieldCacheWithThreads, TestManagedResource, DistanceUnitsTest, BadCopyFieldTest, TestRandomRequestDistribution, WordBreakSolrSpellCheckerTest, TestSolrCloudSnapshots, TestSolrConfigHandlerCloud, CircularListTest, TestSchemaVersionResource, TestExceedMaxTermLength, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, CollectionsAPIDistributedZkTest, BasicZkTest, TestReplicationHandler, TermVectorComponentDistributedTest, TestStressReorder, TestJoin, TestReload, TestStressVersions, HardAutoCommitTest, DistributedTermsComponentTest, SolrCoreTest, ConvertedLegacyTest, TestSort, TestFiltering, TestFunctionQuery, DirectUpdateHandlerTest, TestBadConfig, HighlighterTest, SoftAutoCommitTest, LukeRequestHandlerTest, TestQueryTypes, TermsComponentTest, FastVectorHighlighterTest, SolrPluginUtilsTest, TestCollationField, TestCSVResponseWriter, TestPartialUpdateDeduplication, PingRequestHandlerTest, SearchHandlerTest, HighlighterConfigTest, TestQuerySenderListener, ResponseLogComponentTest, TestStressRecovery, DirectSolrConnectionTest, TestPhraseSuggestions, EchoParamsTest, TestDFRSimilarityFactory, ResourceLoaderTest, URLClassifyProcessorTest, DateMathParserTest, PrimUtilsTest, DateFieldTest, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestUtils, SliceStateTest, SystemInfoHandlerTest, DistributedMLTComponentTest, TestRTGBase, CursorPagingTest, DistributedIntervalFacetingTest, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestSimpleTrackingShardHandler, TestTolerantSearch, TestJettySolrRunner, ConnectionReuseTest, ActionThrottleTest, AliasIntegrationTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrReplicationDistributedZkTest, CdcrReplicationHandlerTest, ChaosMonkeySafeLeaderWithPullReplicasTest, ConfigSetsAPITest, ConnectionManagerTest, CustomCollectionTest, DeleteShardTest, DistribJoinFromCollectionTest, DistributedVersionInfoTest, HttpPartitionTest, LeaderFailureAfterFreshStartTest, MissingSegmentRecoveryTest, MoveReplicaHDFSFailoverTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, PeerSyncReplicationTest, TestCollectionAPI, TestConfigSetsAPI, TestHdfsCloudBackupRestore, TestLockTree, TestOnReconnectListenerSupport, TestPullReplicaErrorHandling, TestReplicaProperties, TestSizeLimitedDistributedMap, TestSolrCloudWithDelegationTokens, TestSolrCloudWithSecureImpersonation]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=6FFD94895A07389C -Dtests.slow=true -Dtests.locale=und -Dtests.timezone=America/North_Dakota/Beulah -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 0.00s J0 | TestSolrCloudWithSecureImpersonation (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
[junit4] > 1) Thread[id=33608, name=jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > 2) Thread[id=33609, name=jetty-launcher-5069-thread-1-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
[junit4] > 3) Thread[id=33548, name=jetty-launcher-5069-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] > at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] > at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] > at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] > at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] > at __randomizedtesting.SeedInfo.seed([6FFD94895A07389C]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=33608, name=jetty-launcher-5069-thread-1-SendThread(127.0.0.1:35405), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > at __randomizedtesting.SeedInfo.seed([6FFD94895A07389C]:0)
[junit4] Completed [570/744 (1!)] on J0 in 66.13s, 12 tests, 2 errors <<< FAILURES!
[...truncated 47586 lines...]
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 244 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/244/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC
3 tests failed.
FAILED: org.apache.solr.cloud.SyncSliceTest.test
Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:44075 within 30000 ms
Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:44075 within 30000 ms
at __randomizedtesting.SeedInfo.seed([7DA440D52DE84075:F5F07F0F83142D8D]:0)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:183)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
at org.apache.solr.cloud.AbstractDistribZkTestBase.printLayout(AbstractDistribZkTestBase.java:323)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.distribTearDown(AbstractFullDistribZkTestBase.java:1553)
at org.apache.solr.cloud.SyncSliceTest.distribTearDown(SyncSliceTest.java:56)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:970)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:44075 within 30000 ms
at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:233)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:175)
... 38 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.SyncSliceTest
Error Message:
62 threads leaked from SUITE scope at org.apache.solr.cloud.SyncSliceTest: 1) Thread[id=12438, name=OverseerThreadFactory-4774-thread-2, state=WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=12443, name=qtp10809358-12443, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=12412, name=TEST-SyncSliceTest.test-seed#[7DA440D52DE84075]-EventThread, state=WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 4) Thread[id=12436, name=TEST-SyncSliceTest.test-seed#[7DA440D52DE84075]-EventThread, state=WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 5) Thread[id=12414, name=OverseerStateUpdate-98836924667002885-127.0.0.1:56261_r_c%2Fhh-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.ZkDistributedQueue.peek(ZkDistributedQueue.java:169) at org.apache.solr.cloud.ZkDistributedQueue.peek(ZkDistributedQueue.java:142) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:192) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=12441, name=qtp10809358-12441, state=RUNNABLE, group=TGRP-SyncSliceTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=12473, name=qtp1193364598-12473, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=12440, name=qtp10809358-12440-acceptor-0@23b5cbdc-ServerConnector@6f787074{SSL,[ssl, http/1.1]}{127.0.0.1:42771}, state=RUNNABLE, group=TGRP-SyncSliceTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=12402, name=org.eclipse.jetty.server.session.HashSessionManager@46c70951Timer, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=12381, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-SyncSliceTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=12401, name=qtp913149588-12401, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=12411, name=TEST-SyncSliceTest.test-seed#[7DA440D52DE84075]-SendThread(127.0.0.1:44075), state=RUNNABLE, group=TGRP-SyncSliceTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141) 13) Thread[id=12469, name=qtp1193364598-12469, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=12400, name=qtp913149588-12400, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=12476, name=qtp1193364598-12476, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=12433, name=Connection evictor, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=12434, name=Connection evictor, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=12451, name=Connection evictor, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=12416, name=OverseerAutoScalingTriggerThread-98836924667002885-127.0.0.1:56261_r_c%2Fhh-n_0000000000, state=WAITING, group=Overseer autoscaling triggers] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:168) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=12407, name=Connection evictor, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=12452, name=Connection evictor, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=12470, name=qtp1193364598-12470-acceptor-0@5fe52d58-ServerConnector@5c77cff3{SSL,[ssl, http/1.1]}{127.0.0.1:44075}, state=RUNNABLE, group=TGRP-SyncSliceTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=12463, name=OverseerThreadFactory-4774-thread-3, state=WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=12444, name=qtp10809358-12444, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=12380, name=Thread-4053, state=WAITING, group=TGRP-SyncSliceTest] at java.lang.Object.wait(Native Method) at java.lang.Thread.join(Thread.java:1252) at java.lang.Thread.join(Thread.java:1326) at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297) at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309) at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490) 26) Thread[id=12466, name=searcherExecutor-4788-thread-1, state=WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=12396, name=qtp913149588-12396, state=RUNNABLE, group=TGRP-SyncSliceTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 28) Thread[id=12456, name=TEST-SyncSliceTest.test-seed#[7DA440D52DE84075]-SendThread(127.0.0.1:44075), state=RUNNABLE, group=TGRP-SyncSliceTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141) 29) Thread[id=12419, name=Thread-4062, state=WAITING, group=TGRP-SyncSliceTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1669) 30) Thread[id=12468, name=Connection evictor, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 31) Thread[id=12406, name=Connection evictor, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=12474, name=qtp1193364598-12474, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=12415, name=OverseerCollectionConfigSetProcessor-98836924667002885-127.0.0.1:56261_r_c%2Fhh-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:365) at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:228) at java.lang.Thread.run(Thread.java:748) 34) Thread[id=12394, name=qtp913149588-12394, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=12417, name=ScheduledTrigger-4772-thread-1, state=WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=12447, name=org.eclipse.jetty.server.session.HashSessionManager@22881cf6Timer, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 37) Thread[id=12480, name=Scheduler-696276214, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 38) Thread[id=12398, name=qtp913149588-12398, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 39) Thread[id=12397, name=qtp913149588-12397, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 40) Thread[id=12384, name=ProcessThread(sid:0 cport:44075):, state=WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122) 41) Thread[id=12472, name=qtp1193364598-12472, state=RUNNABLE, group=TGRP-SyncSliceTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 42) Thread[id=12445, name=qtp10809358-12445, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 43) Thread[id=12427, name=OverseerThreadFactory-4774-thread-1, state=WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 44) Thread[id=12462, name=Scheduler-306498804, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 45) Thread[id=12442, name=qtp10809358-12442, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 46) Thread[id=12471, name=qtp1193364598-12471, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 47) Thread[id=12425, name=Scheduler-216828009, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 48) Thread[id=12467, name=Connection evictor, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 49) Thread[id=12399, name=qtp913149588-12399, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 50) Thread[id=12446, name=qtp10809358-12446, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 51) Thread[id=12418, name=ScheduledTrigger-4772-thread-2, state=WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 52) Thread[id=12395, name=qtp913149588-12395-acceptor-0@7337e0c2-ServerConnector@6bc7b0f{SSL,[ssl, http/1.1]}{127.0.0.1:56261}, state=RUNNABLE, group=TGRP-SyncSliceTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 53) Thread[id=12439, name=qtp10809358-12439, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 54) Thread[id=12426, name=ScheduledTrigger-4772-thread-3, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 55) Thread[id=12457, name=TEST-SyncSliceTest.test-seed#[7DA440D52DE84075]-EventThread, state=WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 56) Thread[id=12432, name=searcherExecutor-4777-thread-1, state=WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 57) Thread[id=12460, name=Thread-4071, state=WAITING, group=TGRP-SyncSliceTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1669) 58) Thread[id=12428, name=ScheduledTrigger-4772-thread-4, state=WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 59) Thread[id=12475, name=qtp1193364598-12475, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 60) Thread[id=12435, name=TEST-SyncSliceTest.test-seed#[7DA440D52DE84075]-SendThread(127.0.0.1:44075), state=RUNNABLE, group=TGRP-SyncSliceTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141) 61) Thread[id=12382, name=SessionTracker, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at java.lang.Object.wait(Native Method) at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146) 62) Thread[id=12477, name=org.eclipse.jetty.server.session.HashSessionManager@5b371becTimer, state=TIMED_WAITING, group=TGRP-SyncSliceTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 62 threads leaked from SUITE scope at org.apache.solr.cloud.SyncSliceTest:
1) Thread[id=12438, name=OverseerThreadFactory-4774-thread-2, state=WAITING, group=Overseer collection creation process.]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=12443, name=qtp10809358-12443, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=12412, name=TEST-SyncSliceTest.test-seed#[7DA440D52DE84075]-EventThread, state=WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
4) Thread[id=12436, name=TEST-SyncSliceTest.test-seed#[7DA440D52DE84075]-EventThread, state=WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
5) Thread[id=12414, name=OverseerStateUpdate-98836924667002885-127.0.0.1:56261_r_c%2Fhh-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.apache.solr.cloud.ZkDistributedQueue.peek(ZkDistributedQueue.java:169)
at org.apache.solr.cloud.ZkDistributedQueue.peek(ZkDistributedQueue.java:142)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:192)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=12441, name=qtp10809358-12441, state=RUNNABLE, group=TGRP-SyncSliceTest]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=12473, name=qtp1193364598-12473, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=12440, name=qtp10809358-12440-acceptor-0@23b5cbdc-ServerConnector@6f787074{SSL,[ssl, http/1.1]}{127.0.0.1:42771}, state=RUNNABLE, group=TGRP-SyncSliceTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=12402, name=org.eclipse.jetty.server.session.HashSessionManager@46c70951Timer, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=12381, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-SyncSliceTest]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=12401, name=qtp913149588-12401, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=12411, name=TEST-SyncSliceTest.test-seed#[7DA440D52DE84075]-SendThread(127.0.0.1:44075), state=RUNNABLE, group=TGRP-SyncSliceTest]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
13) Thread[id=12469, name=qtp1193364598-12469, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=12400, name=qtp913149588-12400, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=12476, name=qtp1193364598-12476, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
16) Thread[id=12433, name=Connection evictor, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
17) Thread[id=12434, name=Connection evictor, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
18) Thread[id=12451, name=Connection evictor, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
19) Thread[id=12416, name=OverseerAutoScalingTriggerThread-98836924667002885-127.0.0.1:56261_r_c%2Fhh-n_0000000000, state=WAITING, group=Overseer autoscaling triggers]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:168)
at java.lang.Thread.run(Thread.java:748)
20) Thread[id=12407, name=Connection evictor, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
21) Thread[id=12452, name=Connection evictor, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
22) Thread[id=12470, name=qtp1193364598-12470-acceptor-0@5fe52d58-ServerConnector@5c77cff3{SSL,[ssl, http/1.1]}{127.0.0.1:44075}, state=RUNNABLE, group=TGRP-SyncSliceTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
23) Thread[id=12463, name=OverseerThreadFactory-4774-thread-3, state=WAITING, group=Overseer collection creation process.]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
24) Thread[id=12444, name=qtp10809358-12444, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
25) Thread[id=12380, name=Thread-4053, state=WAITING, group=TGRP-SyncSliceTest]
at java.lang.Object.wait(Native Method)
at java.lang.Thread.join(Thread.java:1252)
at java.lang.Thread.join(Thread.java:1326)
at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
26) Thread[id=12466, name=searcherExecutor-4788-thread-1, state=WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
27) Thread[id=12396, name=qtp913149588-12396, state=RUNNABLE, group=TGRP-SyncSliceTest]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
28) Thread[id=12456, name=TEST-SyncSliceTest.test-seed#[7DA440D52DE84075]-SendThread(127.0.0.1:44075), state=RUNNABLE, group=TGRP-SyncSliceTest]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
29) Thread[id=12419, name=Thread-4062, state=WAITING, group=TGRP-SyncSliceTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1669)
30) Thread[id=12468, name=Connection evictor, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
31) Thread[id=12406, name=Connection evictor, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
32) Thread[id=12474, name=qtp1193364598-12474, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
33) Thread[id=12415, name=OverseerCollectionConfigSetProcessor-98836924667002885-127.0.0.1:56261_r_c%2Fhh-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:365)
at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)
at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:228)
at java.lang.Thread.run(Thread.java:748)
34) Thread[id=12394, name=qtp913149588-12394, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
35) Thread[id=12417, name=ScheduledTrigger-4772-thread-1, state=WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
36) Thread[id=12447, name=org.eclipse.jetty.server.session.HashSessionManager@22881cf6Timer, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
37) Thread[id=12480, name=Scheduler-696276214, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
38) Thread[id=12398, name=qtp913149588-12398, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
39) Thread[id=12397, name=qtp913149588-12397, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
40) Thread[id=12384, name=ProcessThread(sid:0 cport:44075):, state=WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)
41) Thread[id=12472, name=qtp1193364598-12472, state=RUNNABLE, group=TGRP-SyncSliceTest]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
42) Thread[id=12445, name=qtp10809358-12445, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
43) Thread[id=12427, name=OverseerThreadFactory-4774-thread-1, state=WAITING, group=Overseer collection creation process.]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
44) Thread[id=12462, name=Scheduler-306498804, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
45) Thread[id=12442, name=qtp10809358-12442, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
46) Thread[id=12471, name=qtp1193364598-12471, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
47) Thread[id=12425, name=Scheduler-216828009, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
48) Thread[id=12467, name=Connection evictor, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
49) Thread[id=12399, name=qtp913149588-12399, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
50) Thread[id=12446, name=qtp10809358-12446, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
51) Thread[id=12418, name=ScheduledTrigger-4772-thread-2, state=WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
52) Thread[id=12395, name=qtp913149588-12395-acceptor-0@7337e0c2-ServerConnector@6bc7b0f{SSL,[ssl, http/1.1]}{127.0.0.1:56261}, state=RUNNABLE, group=TGRP-SyncSliceTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
53) Thread[id=12439, name=qtp10809358-12439, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
54) Thread[id=12426, name=ScheduledTrigger-4772-thread-3, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
55) Thread[id=12457, name=TEST-SyncSliceTest.test-seed#[7DA440D52DE84075]-EventThread, state=WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
56) Thread[id=12432, name=searcherExecutor-4777-thread-1, state=WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
57) Thread[id=12460, name=Thread-4071, state=WAITING, group=TGRP-SyncSliceTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1669)
58) Thread[id=12428, name=ScheduledTrigger-4772-thread-4, state=WAITING, group=TGRP-SyncSliceTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThr
[...truncated too long message...]
> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 10) Thread[id=12396, name=qtp913149588-12396, state=RUNNABLE, group=TGRP-SyncSliceTest]
[junit4] > at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
[junit4] > at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
[junit4] > at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 11) Thread[id=12456, name=TEST-SyncSliceTest.test-seed#[7DA440D52DE84075]-SendThread(127.0.0.1:44075), state=TIMED_WAITING, group=TGRP-SyncSliceTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > 12) Thread[id=12432, name=searcherExecutor-4777-thread-1, state=WAITING, group=TGRP-SyncSliceTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 13) Thread[id=12402, name=org.eclipse.jetty.server.session.HashSessionManager@46c70951Timer, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 14) Thread[id=12460, name=Thread-4071, state=WAITING, group=TGRP-SyncSliceTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1669)
[junit4] > 15) Thread[id=12428, name=ScheduledTrigger-4772-thread-4, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 16) Thread[id=12462, name=Scheduler-306498804, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 17) Thread[id=12419, name=Thread-4062, state=WAITING, group=TGRP-SyncSliceTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1669)
[junit4] > 18) Thread[id=12411, name=TEST-SyncSliceTest.test-seed#[7DA440D52DE84075]-SendThread(127.0.0.1:44075), state=TIMED_WAITING, group=TGRP-SyncSliceTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > 19) Thread[id=12435, name=TEST-SyncSliceTest.test-seed#[7DA440D52DE84075]-SendThread(127.0.0.1:44075), state=TIMED_WAITING, group=TGRP-SyncSliceTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > 20) Thread[id=12477, name=org.eclipse.jetty.server.session.HashSessionManager@5b371becTimer, state=TIMED_WAITING, group=TGRP-SyncSliceTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([7DA440D52DE84075]:0)
[junit4] Completed [241/744 (1!)] on J0 in 94.20s, 1 test, 3 errors <<< FAILURES!
[...truncated 48515 lines...]